{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python       \n",
    "# coding: utf-8\n",
    "\n",
    "# importing required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import time\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "from os import path\n",
    "import sklearn\n",
    "from sklearn import metrics, preprocessing\n",
    "from sklearn.metrics import (precision_score, recall_score, f1_score, roc_auc_score, \n",
    "                             roc_curve, auc, confusion_matrix, accuracy_score, \n",
    "                             balanced_accuracy_score, matthews_corrcoef, classification_report)\n",
    "from sklearn.preprocessing import (StandardScaler, OrdinalEncoder, LabelEncoder, \n",
    "                                   MinMaxScaler, OneHotEncoder, Normalizer, \n",
    "                                   MaxAbsScaler, RobustScaler, PowerTransformer, LabelBinarizer)\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, BaggingClassifier\n",
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.datasets import load_iris, make_classification\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "from sklearn.feature_selection import mutual_info_classif\n",
    "from collections import Counter\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from lightgbm import LGBMClassifier\n",
    "import catboost\n",
    "import xgboost as xgb\n",
    "from scipy.stats import mode\n",
    "from tabulate import tabulate\n",
    "import shap\n",
    "import joblib\n",
    "\n",
    "start_program = time.time()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters to use or load models, but yet to be implemented, it is not really functional."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First ensemble with NSL-KDD\n",
    "# Parameters\n",
    "# Few parameters are not fully implemented yet\n",
    "\n",
    "#----------------------------------------------\n",
    "# 0 for not using it as base learner\n",
    "# 1 for using it as base learner\n",
    "# not implemented but in the code in someparts\n",
    "use_model_ada = 1 \n",
    "use_model_dnn = 1 \n",
    "use_model_mlp = 1 \n",
    "use_model_lgbm = 1 \n",
    "use_model_rf = 1 \n",
    "use_model_svm = 1\n",
    "use_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "# 0 for training the model\n",
    "# 1 for using the saved version of the model\n",
    "\n",
    "load_model_ada = 1\n",
    "load_model_dnn = 1 \n",
    "load_model_mlp = 1 \n",
    "load_model_lgbm = 1 \n",
    "load_model_rf = 1                               \n",
    "load_model_svm = 1\n",
    "load_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "\n",
    "# Implemented\n",
    "#----------------------------------------------\n",
    "# feature_selection_bit = 0 # OFF\n",
    "feature_selection_bit = 1# On\n",
    "pick_prob = 1 # set equal one to choose the dataset with probabilities, set to 0 to choose one with the classes.\n",
    "# pick_prob = 0\n",
    "generate_feature_importance = 0 # Generate Shap graphs\n",
    "\n",
    "\n",
    "# choose the features wanted to load.\n",
    "column_features = [\n",
    "                    # 'dnn',\n",
    "                #    'rf',\n",
    "                   'lgbm',\n",
    "                #    'ada',\n",
    "                   'knn',\n",
    "                   'mlp',\n",
    "                   'svm',\n",
    "                #    'cat',\n",
    "                #    'xgb',\n",
    "                   'lr',\n",
    "                   'dt',\n",
    "                   'label']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Pick name of the file according to parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Specify the name of the output text file\n",
    "if feature_selection_bit == 0:\n",
    "\n",
    "    if pick_prob == 0:\n",
    "        output_file_name = \"ensemble_level_01_all_features_classes.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_all_features_classes--', file = f)\n",
    "\n",
    "    elif pick_prob == 1:\n",
    "        output_file_name = \"ensemble_level_01_all_features_probabilites.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_all_features_probabilites--', file = f)\n",
    "\n",
    "elif feature_selection_bit == 1:\n",
    "    if pick_prob == 0:\n",
    "        output_file_name = \"ensemble_level_01_feature_selection_classes.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_feature_selection_classes--', file = f)\n",
    "    elif pick_prob == 1:\n",
    "        output_file_name = \"ensemble_level_01_feature_selection_probabilites.txt\"\n",
    "        with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "        with open(output_file_name, \"a\") as f: print('----ensemble_level_01_feature_selection_probabilites--', file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_program = time.time()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def confusion_metrics (name_model,predictions,true_labels):\n",
    "\n",
    "    name = name_model\n",
    "    pred_label = predictions\n",
    "    y_test_01 = true_labels \n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(name, file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "    return Acc, Precision, Recall, F1, BACC, MCC\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load created datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df_level_00_1=pd.read_csv('base_models_prob_feature_selection.csv')\n",
    "df_level_00_0=pd.read_csv('base_models_class_feature_selection.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.921925</td>\n",
       "      <td>0.962559</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.315597</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999989</td>\n",
       "      <td>0.009174</td>\n",
       "      <td>0.995496</td>\n",
       "      <td>0.994228</td>\n",
       "      <td>0.997583</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.916570</td>\n",
       "      <td>0.962559</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.323321</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.506630</td>\n",
       "      <td>0.995372</td>\n",
       "      <td>0.993423</td>\n",
       "      <td>0.999931</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.931765</td>\n",
       "      <td>0.960068</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.312277</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999954</td>\n",
       "      <td>0.891043</td>\n",
       "      <td>0.994298</td>\n",
       "      <td>0.993254</td>\n",
       "      <td>0.993783</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.571414</td>\n",
       "      <td>0.987811</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.283245</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.507230</td>\n",
       "      <td>0.996083</td>\n",
       "      <td>0.997760</td>\n",
       "      <td>0.999845</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.570172</td>\n",
       "      <td>0.987811</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.283245</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.992353</td>\n",
       "      <td>0.995679</td>\n",
       "      <td>0.997760</td>\n",
       "      <td>0.999840</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4495</th>\n",
       "      <td>0.563370</td>\n",
       "      <td>0.829673</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.281672</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.140634</td>\n",
       "      <td>0.982723</td>\n",
       "      <td>0.991977</td>\n",
       "      <td>0.998422</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4496</th>\n",
       "      <td>0.537440</td>\n",
       "      <td>0.987811</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.292088</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.052616</td>\n",
       "      <td>0.995899</td>\n",
       "      <td>0.997760</td>\n",
       "      <td>0.999914</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4497</th>\n",
       "      <td>0.476535</td>\n",
       "      <td>0.880875</td>\n",
       "      <td>0.999988</td>\n",
       "      <td>0.307149</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.996455</td>\n",
       "      <td>0.783083</td>\n",
       "      <td>0.914719</td>\n",
       "      <td>0.948133</td>\n",
       "      <td>0.823161</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4498</th>\n",
       "      <td>0.812448</td>\n",
       "      <td>0.923602</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.342729</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.836484</td>\n",
       "      <td>0.990981</td>\n",
       "      <td>0.983675</td>\n",
       "      <td>0.993822</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4499</th>\n",
       "      <td>0.553209</td>\n",
       "      <td>0.988558</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.243645</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.047286</td>\n",
       "      <td>0.996039</td>\n",
       "      <td>0.997760</td>\n",
       "      <td>0.999677</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4500 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           dnn        rf      lgbm       ada  knn       mlp       svm  \\\n",
       "0     0.921925  0.962559  1.000000  0.315597  1.0  0.999989  0.009174   \n",
       "1     0.916570  0.962559  1.000000  0.323321  1.0  1.000000  0.506630   \n",
       "2     0.931765  0.960068  0.999999  0.312277  1.0  0.999954  0.891043   \n",
       "3     0.571414  0.987811  1.000000  0.283245  1.0  1.000000  0.507230   \n",
       "4     0.570172  0.987811  1.000000  0.283245  1.0  1.000000  0.992353   \n",
       "...        ...       ...       ...       ...  ...       ...       ...   \n",
       "4495  0.563370  0.829673  0.999999  0.281672  1.0  1.000000  0.140634   \n",
       "4496  0.537440  0.987811  1.000000  0.292088  1.0  0.999999  0.052616   \n",
       "4497  0.476535  0.880875  0.999988  0.307149  1.0  0.996455  0.783083   \n",
       "4498  0.812448  0.923602  0.999999  0.342729  1.0  1.000000  0.836484   \n",
       "4499  0.553209  0.988558  1.000000  0.243645  1.0  1.000000  0.047286   \n",
       "\n",
       "           cat       xgb        lr   dt  label  \n",
       "0     0.995496  0.994228  0.997583  1.0    0.0  \n",
       "1     0.995372  0.993423  0.999931  1.0    0.0  \n",
       "2     0.994298  0.993254  0.993783  1.0    0.0  \n",
       "3     0.996083  0.997760  0.999845  1.0    1.0  \n",
       "4     0.995679  0.997760  0.999840  1.0    1.0  \n",
       "...        ...       ...       ...  ...    ...  \n",
       "4495  0.982723  0.991977  0.998422  1.0    1.0  \n",
       "4496  0.995899  0.997760  0.999914  1.0    1.0  \n",
       "4497  0.914719  0.948133  0.823161  1.0    0.0  \n",
       "4498  0.990981  0.983675  0.993822  1.0    0.0  \n",
       "4499  0.996039  0.997760  0.999677  1.0    1.0  \n",
       "\n",
       "[4500 rows x 12 columns]"
      ]
     },
     "execution_count": 323,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [],
   "source": [
    "y1 = df_level_00_1.pop('label')\n",
    "X1 = df_level_00_1\n",
    "df_level_00_1 = X1.assign(label = y1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [],
   "source": [
    "y0 = df_level_00_0.pop('label')\n",
    "X0 = df_level_00_0\n",
    "df_level_00_0 = X0.assign(label = y0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Toggle the bit to 1 if feature select is wanted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lgbm    0.993279\n",
      "xgb     0.972628\n",
      "dt      0.956487\n",
      "cat     0.949534\n",
      "mlp     0.947628\n",
      "knn     0.895586\n",
      "lr      0.876125\n",
      "svm     0.848745\n",
      "rf      0.778075\n",
      "dnn     0.551505\n",
      "ada     0.492850\n",
      "dtype: float64\n",
      "rf      0.866737\n",
      "ada     0.845186\n",
      "xgb     0.743265\n",
      "dnn     0.699221\n",
      "cat     0.476201\n",
      "lgbm    0.446241\n",
      "svm     0.342896\n",
      "mlp     0.186961\n",
      "lr      0.170701\n",
      "knn     0.012751\n",
      "dt      0.004745\n",
      "dtype: float64\n",
      "Top 5 feature names:\n",
      "['lgbm', 'xgb', 'dt', 'cat', 'mlp']\n",
      "['rf', 'ada', 'xgb', 'dnn', 'cat']\n"
     ]
    }
   ],
   "source": [
    "if feature_selection_bit == 1:\n",
    "\n",
    "    from sklearn.feature_selection import mutual_info_classif\n",
    "    %matplotlib inline\n",
    "\n",
    "    # Compute information gain using mutual information\n",
    "    importances0 = mutual_info_classif(X0, y0)\n",
    "    importances1 = mutual_info_classif(X1, y1)\n",
    "\n",
    "\n",
    "    feat_importances0 = pd.Series(importances0, df_level_00_0.columns[0:len(df_level_00_0.columns)-1])\n",
    "    feat_importances1= pd.Series(importances1, df_level_00_1.columns[0:len(df_level_00_1.columns)-1])\n",
    "\n",
    "    # feat_importances.plot(kind='barh', color = 'teal')\n",
    "    feat_importances_sorted0 = feat_importances0.sort_values( ascending=False)\n",
    "    feat_importances_sorted1 = feat_importances1.sort_values( ascending=False)\n",
    "\n",
    "\n",
    "    # Print or use the sorted DataFrame\n",
    "    print(feat_importances_sorted0)\n",
    "    print(feat_importances_sorted1)\n",
    "\n",
    "    # feat_importances_sorted.plot(kind='barh', color = 'teal')\n",
    "    # feat_importances_sorted\n",
    "    top_features0 = feat_importances_sorted0.nlargest(5)\n",
    "    top_features1 = feat_importances_sorted1.nlargest(5)\n",
    "\n",
    "    top_feature_names0 = top_features0.index.tolist()\n",
    "    top_feature_names1 = top_features1.index.tolist()\n",
    "\n",
    "\n",
    "    print(\"Top 5 feature names:\")\n",
    "    print(top_feature_names0)\n",
    "    print(top_feature_names1)\n",
    "\n",
    "    column_features0 = top_feature_names0\n",
    "    column_features1 = top_feature_names1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "if feature_selection_bit == 1:\n",
    "    df_level_00_0 = df_level_00_0[column_features0]\n",
    "    df_level_00_1 = df_level_00_1[column_features1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 328,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rf</th>\n",
       "      <th>ada</th>\n",
       "      <th>xgb</th>\n",
       "      <th>dnn</th>\n",
       "      <th>cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.962559</td>\n",
       "      <td>0.315597</td>\n",
       "      <td>0.994228</td>\n",
       "      <td>0.921925</td>\n",
       "      <td>0.995496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.962559</td>\n",
       "      <td>0.323321</td>\n",
       "      <td>0.993423</td>\n",
       "      <td>0.916570</td>\n",
       "      <td>0.995372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.960068</td>\n",
       "      <td>0.312277</td>\n",
       "      <td>0.993254</td>\n",
       "      <td>0.931765</td>\n",
       "      <td>0.994298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.987811</td>\n",
       "      <td>0.283245</td>\n",
       "      <td>0.997760</td>\n",
       "      <td>0.571414</td>\n",
       "      <td>0.996083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.987811</td>\n",
       "      <td>0.283245</td>\n",
       "      <td>0.997760</td>\n",
       "      <td>0.570172</td>\n",
       "      <td>0.995679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4495</th>\n",
       "      <td>0.829673</td>\n",
       "      <td>0.281672</td>\n",
       "      <td>0.991977</td>\n",
       "      <td>0.563370</td>\n",
       "      <td>0.982723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4496</th>\n",
       "      <td>0.987811</td>\n",
       "      <td>0.292088</td>\n",
       "      <td>0.997760</td>\n",
       "      <td>0.537440</td>\n",
       "      <td>0.995899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4497</th>\n",
       "      <td>0.880875</td>\n",
       "      <td>0.307149</td>\n",
       "      <td>0.948133</td>\n",
       "      <td>0.476535</td>\n",
       "      <td>0.914719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4498</th>\n",
       "      <td>0.923602</td>\n",
       "      <td>0.342729</td>\n",
       "      <td>0.983675</td>\n",
       "      <td>0.812448</td>\n",
       "      <td>0.990981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4499</th>\n",
       "      <td>0.988558</td>\n",
       "      <td>0.243645</td>\n",
       "      <td>0.997760</td>\n",
       "      <td>0.553209</td>\n",
       "      <td>0.996039</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4500 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            rf       ada       xgb       dnn       cat\n",
       "0     0.962559  0.315597  0.994228  0.921925  0.995496\n",
       "1     0.962559  0.323321  0.993423  0.916570  0.995372\n",
       "2     0.960068  0.312277  0.993254  0.931765  0.994298\n",
       "3     0.987811  0.283245  0.997760  0.571414  0.996083\n",
       "4     0.987811  0.283245  0.997760  0.570172  0.995679\n",
       "...        ...       ...       ...       ...       ...\n",
       "4495  0.829673  0.281672  0.991977  0.563370  0.982723\n",
       "4496  0.987811  0.292088  0.997760  0.537440  0.995899\n",
       "4497  0.880875  0.307149  0.948133  0.476535  0.914719\n",
       "4498  0.923602  0.342729  0.983675  0.812448  0.990981\n",
       "4499  0.988558  0.243645  0.997760  0.553209  0.996039\n",
       "\n",
       "[4500 rows x 5 columns]"
      ]
     },
     "execution_count": 328,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lgbm</th>\n",
       "      <th>xgb</th>\n",
       "      <th>dt</th>\n",
       "      <th>cat</th>\n",
       "      <th>mlp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4495</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4496</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4497</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4498</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4499</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4500 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      lgbm  xgb   dt  cat  mlp\n",
       "0      0.0  0.0  0.0  0.0  0.0\n",
       "1      0.0  0.0  0.0  0.0  0.0\n",
       "2      0.0  0.0  0.0  0.0  0.0\n",
       "3      1.0  1.0  1.0  1.0  1.0\n",
       "4      1.0  1.0  1.0  1.0  1.0\n",
       "...    ...  ...  ...  ...  ...\n",
       "4495   1.0  1.0  1.0  1.0  1.0\n",
       "4496   1.0  1.0  1.0  1.0  1.0\n",
       "4497   0.0  0.0  0.0  0.0  0.0\n",
       "4498   0.0  0.0  0.0  0.0  0.0\n",
       "4499   1.0  1.0  1.0  1.0  1.0\n",
       "\n",
       "[4500 rows x 5 columns]"
      ]
     },
     "execution_count": 329,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### If the probabilities option is wanted select 1, if it is classses select 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {},
   "outputs": [],
   "source": [
    "if pick_prob == 1:\n",
    "    df_level_01 = df_level_00_1\n",
    "else: \n",
    "    df_level_01 = df_level_00_0\n",
    "\n",
    "df_level_01 = df_level_01.assign(label = y1)\n",
    "\n",
    "y_01 = df_level_01.pop('label') \n",
    "    \n",
    "X_01 = df_level_01\n",
    "df_level_01 = df_level_01.assign(label = y_01)\n",
    "\n",
    "split = 0.7\n",
    "X_train_01,X_test_01, y_train_01, y_test_01 = sklearn.model_selection.train_test_split(X_01, y_01, train_size=split)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the stronger model - STACK level 01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {},
   "outputs": [],
   "source": [
    "#----------------------------------------------------------------\n",
    "with open(output_file_name, \"a\") as f: print('Stack model - Strong learner - level 01', file = f)\n",
    "with open(output_file_name, \"a\") as f: print('-------------------------------------------------------', file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rf</th>\n",
       "      <th>ada</th>\n",
       "      <th>xgb</th>\n",
       "      <th>dnn</th>\n",
       "      <th>cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>947</th>\n",
       "      <td>0.960068</td>\n",
       "      <td>0.315597</td>\n",
       "      <td>0.994228</td>\n",
       "      <td>0.937316</td>\n",
       "      <td>0.994870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308</th>\n",
       "      <td>0.923602</td>\n",
       "      <td>0.329028</td>\n",
       "      <td>0.982133</td>\n",
       "      <td>0.848824</td>\n",
       "      <td>0.992103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2607</th>\n",
       "      <td>0.980553</td>\n",
       "      <td>0.285100</td>\n",
       "      <td>0.997526</td>\n",
       "      <td>0.538876</td>\n",
       "      <td>0.993727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>779</th>\n",
       "      <td>0.303978</td>\n",
       "      <td>0.355229</td>\n",
       "      <td>0.983888</td>\n",
       "      <td>0.183852</td>\n",
       "      <td>0.932000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1063</th>\n",
       "      <td>0.655765</td>\n",
       "      <td>0.314990</td>\n",
       "      <td>0.890187</td>\n",
       "      <td>0.745144</td>\n",
       "      <td>0.968346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1855</th>\n",
       "      <td>0.871801</td>\n",
       "      <td>0.302229</td>\n",
       "      <td>0.987991</td>\n",
       "      <td>0.431270</td>\n",
       "      <td>0.978155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3627</th>\n",
       "      <td>0.369449</td>\n",
       "      <td>0.280640</td>\n",
       "      <td>0.976961</td>\n",
       "      <td>0.022077</td>\n",
       "      <td>0.828767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1287</th>\n",
       "      <td>0.987811</td>\n",
       "      <td>0.283245</td>\n",
       "      <td>0.997760</td>\n",
       "      <td>0.573287</td>\n",
       "      <td>0.995679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1446</th>\n",
       "      <td>0.962559</td>\n",
       "      <td>0.307769</td>\n",
       "      <td>0.993254</td>\n",
       "      <td>0.928510</td>\n",
       "      <td>0.994358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1120</th>\n",
       "      <td>0.619872</td>\n",
       "      <td>0.290535</td>\n",
       "      <td>0.886541</td>\n",
       "      <td>0.953510</td>\n",
       "      <td>0.606012</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1350 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            rf       ada       xgb       dnn       cat\n",
       "947   0.960068  0.315597  0.994228  0.937316  0.994870\n",
       "308   0.923602  0.329028  0.982133  0.848824  0.992103\n",
       "2607  0.980553  0.285100  0.997526  0.538876  0.993727\n",
       "779   0.303978  0.355229  0.983888  0.183852  0.932000\n",
       "1063  0.655765  0.314990  0.890187  0.745144  0.968346\n",
       "...        ...       ...       ...       ...       ...\n",
       "1855  0.871801  0.302229  0.987991  0.431270  0.978155\n",
       "3627  0.369449  0.280640  0.976961  0.022077  0.828767\n",
       "1287  0.987811  0.283245  0.997760  0.573287  0.995679\n",
       "1446  0.962559  0.307769  0.993254  0.928510  0.994358\n",
       "1120  0.619872  0.290535  0.886541  0.953510  0.606012\n",
       "\n",
       "[1350 rows x 5 columns]"
      ]
     },
     "execution_count": 332,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_01"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0    2.0   3.0  4.0\n",
      "0.0  691.0    7.0    8.0   1.0  1.0\n",
      "1.0    6.0  470.0    2.0   0.0  0.0\n",
      "2.0    5.0    6.0  102.0   4.0  1.0\n",
      "3.0    0.0    0.0    2.0  43.0  1.0\n",
      "4.0    0.0    0.0    0.0   0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9674074074074074\n",
      "Precision total:  0.7495971091794891\n",
      "Recall total:  0.7516883374496002\n",
      "F1 total:  0.7504987137135265\n",
      "BACC total:  0.9396104218120002\n",
      "MCC total:  0.944987840594813\n",
      "0.05147409439086914\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "y_pred contains classes not in y_true\n"
     ]
    }
   ],
   "source": [
    "\n",
    "start = time.time()\n",
    "\n",
    "# Create a Decision Tree Classifier\n",
    "dt_classifier = DecisionTreeClassifier(random_state=42)\n",
    "# Train the classifier on the training data\n",
    "dt_classifier.fit(X_train_01, y_train_01)\n",
    "# Make predictions on the test data\n",
    "preds_dt = dt_classifier.predict(X_test_01)\n",
    "# Evaluate the accuracy of the model\n",
    "preds_dt_prob = dt_classifier.predict_proba(X_test_01)\n",
    "\n",
    "pred_label = preds_dt\n",
    "name = 'dt'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "print(time_taken)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "    \n",
    "if pick_prob == 0:\n",
    "    # Voting start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    df = X_test_01\n",
    "    # Extract predictions columns\n",
    "    \n",
    "   \n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] \n",
    "\n",
    "    # Use the mode function along axis 1 to get the most common prediction for each row\n",
    "    ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "    # Add the ensemble predictions to the DataFrame\n",
    "    df['ensemble'] = ensemble_predictions.astype(int)\n",
    "\n",
    "    # Display the DataFrame with ensemble predictions\n",
    "    print(df)\n",
    "\n",
    "    pred_label = df ['ensemble'].values\n",
    "    df.pop('ensemble')\n",
    "\n",
    "    #testing metrics def\n",
    "    name = 'voting'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "   \n",
    "else:\n",
    "    name = 'voting'\n",
    "    globals()[f\"{name}_acc_01\"] = 0\n",
    "    globals()[f\"{name}_pre_01\"] = 0\n",
    "    globals()[f\"{name}_rec_01\"] = 0\n",
    "    globals()[f\"{name}_f1_01\"] = 0\n",
    "    globals()[f\"{name}_bacc_01\"] = 0\n",
    "    globals()[f\"{name}_mcc_01\"] = 0\n",
    "    globals()[f\"{name}_time_01\"] = 9999\n",
    "   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            rf       ada       xgb       dnn       cat  results\n",
      "947   0.960068  0.315597  0.994228  0.937316  0.994870        1\n",
      "308   0.923602  0.329028  0.982133  0.848824  0.992103        1\n",
      "2607  0.980553  0.285100  0.997526  0.538876  0.993727        1\n",
      "779   0.303978  0.355229  0.983888  0.183852  0.932000        1\n",
      "1063  0.655765  0.314990  0.890187  0.745144  0.968346        1\n",
      "...        ...       ...       ...       ...       ...      ...\n",
      "1855  0.871801  0.302229  0.987991  0.431270  0.978155        1\n",
      "3627  0.369449  0.280640  0.976961  0.022077  0.828767        0\n",
      "1287  0.987811  0.283245  0.997760  0.573287  0.995679        1\n",
      "1446  0.962559  0.307769  0.993254  0.928510  0.994358        1\n",
      "1120  0.619872  0.290535  0.886541  0.953510  0.606012        1\n",
      "\n",
      "[1350 rows x 6 columns]\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "      0.0    1.0  2.0  3.0\n",
      "0.0   8.0  700.0  0.0  0.0\n",
      "1.0   7.0  471.0  0.0  0.0\n",
      "2.0   9.0  109.0  0.0  0.0\n",
      "3.0  31.0   15.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.3548148148148148\n",
      "Precision total:  0.12729027729027728\n",
      "Recall total:  0.24916377089095337\n",
      "F1 total:  0.13806818307819568\n",
      "BACC total:  0.24916377089095337\n",
      "MCC total:  -0.028853429468060625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "if 0 == 0:\n",
    "    # Average start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "    df = X_test_01\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] \n",
    "   \n",
    "\n",
    "    column_sums = df.sum(axis=1)\n",
    "    row_average = df.mean(axis=1)\n",
    "\n",
    "    # Approximate the result to the closest integer\n",
    "    rounded_average = row_average.round().astype(int)\n",
    "\n",
    "    df['results'] = rounded_average\n",
    "    print(df)\n",
    " \n",
    "    pred_label = df ['results'].values\n",
    "    df.pop('results')\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    name = 'avg'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    \n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Weighed Average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "try:\n",
    "    if pick_prob == 1:\n",
    "        column_features = column_features1\n",
    "    else: column_features = column_features0\n",
    "except:\n",
    "    None\n",
    "feature_selection_columns_in_order_of_importance = column_features[:-1]\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.0, 0.3333333333333333, 0.6666666666666666, 1.0]\n",
      "[0.0, 0.3333333333333333, 0.6666666666666666, 1.0]\n",
      "            rf       ada       xgb       dnn\n",
      "947   0.960068  0.315597  0.994228  0.937316\n",
      "308   0.923602  0.329028  0.982133  0.848824\n",
      "2607  0.980553  0.285100  0.997526  0.538876\n",
      "779   0.303978  0.355229  0.983888  0.183852\n",
      "1063  0.655765  0.314990  0.890187  0.745144\n",
      "...        ...       ...       ...       ...\n",
      "1855  0.871801  0.302229  0.987991  0.431270\n",
      "3627  0.369449  0.280640  0.976961  0.022077\n",
      "1287  0.987811  0.283245  0.997760  0.573287\n",
      "1446  0.962559  0.307769  0.993254  0.928510\n",
      "1120  0.619872  0.290535  0.886541  0.953510\n",
      "\n",
      "[1350 rows x 4 columns]\n",
      "947     0.852667\n",
      "308     0.806627\n",
      "2607    0.649463\n",
      "779     0.479094\n",
      "1063    0.721799\n",
      "          ...   \n",
      "1855    0.595337\n",
      "3627    0.383465\n",
      "1287    0.666438\n",
      "1446    0.846635\n",
      "1120    0.820691\n",
      "Length: 1350, dtype: float64\n",
      "947     1\n",
      "308     1\n",
      "2607    1\n",
      "779     0\n",
      "1063    1\n",
      "       ..\n",
      "1855    1\n",
      "3627    0\n",
      "1287    1\n",
      "1446    1\n",
      "1120    1\n",
      "Length: 1350, dtype: int64\n",
      "            rf       ada       xgb       dnn  results\n",
      "947   0.960068  0.315597  0.994228  0.937316        1\n",
      "308   0.923602  0.329028  0.982133  0.848824        1\n",
      "2607  0.980553  0.285100  0.997526  0.538876        1\n",
      "779   0.303978  0.355229  0.983888  0.183852        0\n",
      "1063  0.655765  0.314990  0.890187  0.745144        1\n",
      "...        ...       ...       ...       ...      ...\n",
      "1855  0.871801  0.302229  0.987991  0.431270        1\n",
      "3627  0.369449  0.280640  0.976961  0.022077        0\n",
      "1287  0.987811  0.283245  0.997760  0.573287        1\n",
      "1446  0.962559  0.307769  0.993254  0.928510        1\n",
      "1120  0.619872  0.290535  0.886541  0.953510        1\n",
      "\n",
      "[1350 rows x 5 columns]\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      0.0    1.0  2.0  3.0\n",
      "0.0  18.0  690.0  0.0  0.0\n",
      "1.0  19.0  459.0  0.0  0.0\n",
      "2.0  29.0   89.0  0.0  0.0\n",
      "3.0  46.0    0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.35333333333333333\n",
      "Precision total:  0.13286839372259404\n",
      "Recall total:  0.246418693709666\n",
      "F1 total:  0.14471686849735632\n",
      "BACC total:  0.246418693709666\n",
      "MCC total:  -0.049612990936082714\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "if 0 == 0:\n",
    "    # Average start\n",
    "\n",
    "    import pandas as pd\n",
    "    from scipy.stats import mode\n",
    "\n",
    "\n",
    "    df = X_test_01[feature_selection_columns_in_order_of_importance]\n",
    "    # Extract predictions columns\n",
    "    predictions = df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "    # weight\n",
    "    weights_values = []\n",
    "\n",
    "    # linear weight distribution\n",
    "    for i in range(0,len(~df.columns.isin(['label']))):\n",
    "        weights_values.append(i/(len(~df.columns.isin(['label']))-1))\n",
    "    print(weights_values)\n",
    "    # weights_values = [10,3,2,2.3]\n",
    "    print(weights_values)\n",
    "    print(df)\n",
    "    weighted_average = df.multiply(weights_values).sum(axis=1) / sum(weights_values)\n",
    "    print(weighted_average)\n",
    "    # Approximate the result to the closest integer\n",
    "    rounded_weighted_average = weighted_average.round().astype(int)\n",
    "\n",
    "    print(rounded_weighted_average)\n",
    "\n",
    "    df['results'] = rounded_weighted_average\n",
    "    print(df)\n",
    " \n",
    "    pred_label = df ['results'].values\n",
    "\n",
    "    df.pop('results')\n",
    "\n",
    "    #testing metrics def\n",
    "    name = 'weighed_avg'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging  with DT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0    2.0   3.0  4.0\n",
      "0.0  694.0    2.0   10.0   1.0  1.0\n",
      "1.0    4.0  471.0    3.0   0.0  0.0\n",
      "2.0    2.0    5.0  109.0   1.0  1.0\n",
      "3.0    0.0    1.0    2.0  43.0  0.0\n",
      "4.0    0.0    0.0    0.0   0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9755555555555555\n",
      "Precision total:  0.7618629847341546\n",
      "Recall total:  0.7648186118982208\n",
      "F1 total:  0.7632005729380337\n",
      "BACC total:  0.956023264872776\n",
      "MCC total:  0.9588523534695824\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "y_pred contains classes not in y_true\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "start = time.time()\n",
    "base_classifier = DecisionTreeClassifier(random_state=42)\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_dt'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging  with SVM\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0   2.0   3.0\n",
      "0.0  618.0   82.0   6.0   2.0\n",
      "1.0   18.0  449.0   7.0   4.0\n",
      "2.0   26.0   51.0  31.0  10.0\n",
      "3.0    0.0   15.0   0.0  31.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8362962962962963\n",
      "Precision total:  0.7624371170445091\n",
      "Recall total:  0.6872092019375746\n",
      "F1 total:  0.6967303335701458\n",
      "BACC total:  0.6872092019375746\n",
      "MCC total:  0.7265015640445596\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "svm_01 = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "# # Define the base classifier (Decision Tree in this case)\n",
    "base_classifier = svm_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_svm'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging with MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 340,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0   2.0   3.0  4.0\n",
      "0.0  634.0   62.0  11.0   0.0  1.0\n",
      "1.0    6.0  445.0  17.0  10.0  0.0\n",
      "2.0   20.0   43.0  50.0   5.0  0.0\n",
      "3.0    0.0    0.0   0.0  46.0  0.0\n",
      "4.0    0.0    0.0   0.0   0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8703703703703703\n",
      "Precision total:  0.6329641942756696\n",
      "Recall total:  0.6500342765288514\n",
      "F1 total:  0.6325353010807875\n",
      "BACC total:  0.8125428456610642\n",
      "MCC total:  0.7846958164611808\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "y_pred contains classes not in y_true\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# create MLPClassifier instance\n",
    "mlp_01 = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "base_classifier = mlp_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_mlp'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging knn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 341,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0   2.0   3.0\n",
      "0.0  688.0    3.0  15.0   2.0\n",
      "1.0   10.0  451.0  16.0   1.0\n",
      "2.0   14.0    4.0  92.0   8.0\n",
      "3.0    0.0    0.0   4.0  42.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.942962962962963\n",
      "Precision total:  0.8669676427610695\n",
      "Recall total:  0.9019926379977163\n",
      "F1 total:  0.8830486387075925\n",
      "BACC total:  0.9019926379977163\n",
      "MCC total:  0.9041709349963909\n"
     ]
    }
   ],
   "source": [
    "knn_01=KNeighborsClassifier(n_neighbors = 5)\n",
    "start = time.time()\n",
    "\n",
    "base_classifier = knn_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_knn'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging LogRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining baggin Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0   2.0   3.0\n",
      "0.0  625.0   74.0   7.0   2.0\n",
      "1.0   13.0  445.0  15.0   5.0\n",
      "2.0   24.0   50.0  38.0   6.0\n",
      "3.0    0.0    2.0   0.0  44.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8533333333333334\n",
      "Precision total:  0.7821766050249246\n",
      "Recall total:  0.7730715855284187\n",
      "F1 total:  0.7605427641752488\n",
      "BACC total:  0.7730715855284187\n",
      "MCC total:  0.7555371767540533\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining baggin Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_01 = LogisticRegression()\n",
    "\n",
    "\n",
    "base_classifier = logreg_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_01, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lr'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging ADA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0   1.0   2.0   3.0\n",
      "0.0  642.0  58.0   6.0   2.0\n",
      "1.0  400.0  66.0  12.0   0.0\n",
      "2.0   35.0  58.0  22.0   3.0\n",
      "3.0    0.0   0.0   5.0  41.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.5711111111111111\n",
      "Precision total:  0.5847327194759676\n",
      "Recall total:  0.5306500001541673\n",
      "F1 total:  0.520142682306318\n",
      "BACC total:  0.5306500001541673\n",
      "MCC total:  0.22413722600921565\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "ada = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "base_classifier = ada\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_ada'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging CAT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.3321561\ttotal: 7.35ms\tremaining: 727ms\n",
      "1:\tlearn: 1.1460055\ttotal: 13.2ms\tremaining: 645ms\n",
      "2:\tlearn: 1.0076801\ttotal: 18.5ms\tremaining: 599ms\n",
      "3:\tlearn: 0.8942871\ttotal: 24.2ms\tremaining: 580ms\n",
      "4:\tlearn: 0.8062557\ttotal: 29.8ms\tremaining: 566ms\n",
      "5:\tlearn: 0.7359374\ttotal: 35ms\tremaining: 548ms\n",
      "6:\tlearn: 0.6730170\ttotal: 40.8ms\tremaining: 542ms\n",
      "7:\tlearn: 0.6202162\ttotal: 46.2ms\tremaining: 531ms\n",
      "8:\tlearn: 0.5713458\ttotal: 52.5ms\tremaining: 531ms\n",
      "9:\tlearn: 0.5283769\ttotal: 58.3ms\tremaining: 525ms\n",
      "10:\tlearn: 0.4895663\ttotal: 63.9ms\tremaining: 517ms\n",
      "11:\tlearn: 0.4569699\ttotal: 68.9ms\tremaining: 505ms\n",
      "12:\tlearn: 0.4245054\ttotal: 73.9ms\tremaining: 494ms\n",
      "13:\tlearn: 0.3996663\ttotal: 79.3ms\tremaining: 487ms\n",
      "14:\tlearn: 0.3775880\ttotal: 85.4ms\tremaining: 484ms\n",
      "15:\tlearn: 0.3568403\ttotal: 91.4ms\tremaining: 480ms\n",
      "16:\tlearn: 0.3364731\ttotal: 97.2ms\tremaining: 475ms\n",
      "17:\tlearn: 0.3189125\ttotal: 103ms\tremaining: 469ms\n",
      "18:\tlearn: 0.3020466\ttotal: 109ms\tremaining: 465ms\n",
      "19:\tlearn: 0.2863941\ttotal: 115ms\tremaining: 461ms\n",
      "20:\tlearn: 0.2753088\ttotal: 121ms\tremaining: 456ms\n",
      "21:\tlearn: 0.2639910\ttotal: 128ms\tremaining: 455ms\n",
      "22:\tlearn: 0.2532448\ttotal: 134ms\tremaining: 449ms\n",
      "23:\tlearn: 0.2421840\ttotal: 140ms\tremaining: 444ms\n",
      "24:\tlearn: 0.2325394\ttotal: 146ms\tremaining: 439ms\n",
      "25:\tlearn: 0.2227588\ttotal: 151ms\tremaining: 431ms\n",
      "26:\tlearn: 0.2138240\ttotal: 158ms\tremaining: 426ms\n",
      "27:\tlearn: 0.2055282\ttotal: 163ms\tremaining: 418ms\n",
      "28:\tlearn: 0.1991615\ttotal: 169ms\tremaining: 414ms\n",
      "29:\tlearn: 0.1927194\ttotal: 174ms\tremaining: 406ms\n",
      "30:\tlearn: 0.1880743\ttotal: 180ms\tremaining: 401ms\n",
      "31:\tlearn: 0.1828433\ttotal: 186ms\tremaining: 396ms\n",
      "32:\tlearn: 0.1771459\ttotal: 191ms\tremaining: 389ms\n",
      "33:\tlearn: 0.1727492\ttotal: 197ms\tremaining: 383ms\n",
      "34:\tlearn: 0.1681891\ttotal: 202ms\tremaining: 375ms\n",
      "35:\tlearn: 0.1624658\ttotal: 208ms\tremaining: 370ms\n",
      "36:\tlearn: 0.1598775\ttotal: 213ms\tremaining: 363ms\n",
      "37:\tlearn: 0.1561858\ttotal: 219ms\tremaining: 357ms\n",
      "38:\tlearn: 0.1521693\ttotal: 224ms\tremaining: 350ms\n",
      "39:\tlearn: 0.1483001\ttotal: 230ms\tremaining: 345ms\n",
      "40:\tlearn: 0.1446056\ttotal: 236ms\tremaining: 340ms\n",
      "41:\tlearn: 0.1422636\ttotal: 242ms\tremaining: 334ms\n",
      "42:\tlearn: 0.1398749\ttotal: 248ms\tremaining: 328ms\n",
      "43:\tlearn: 0.1372457\ttotal: 253ms\tremaining: 322ms\n",
      "44:\tlearn: 0.1354138\ttotal: 259ms\tremaining: 316ms\n",
      "45:\tlearn: 0.1329297\ttotal: 264ms\tremaining: 310ms\n",
      "46:\tlearn: 0.1305614\ttotal: 269ms\tremaining: 303ms\n",
      "47:\tlearn: 0.1286038\ttotal: 275ms\tremaining: 298ms\n",
      "48:\tlearn: 0.1265107\ttotal: 281ms\tremaining: 293ms\n",
      "49:\tlearn: 0.1248766\ttotal: 288ms\tremaining: 288ms\n",
      "50:\tlearn: 0.1231727\ttotal: 293ms\tremaining: 282ms\n",
      "51:\tlearn: 0.1210489\ttotal: 299ms\tremaining: 276ms\n",
      "52:\tlearn: 0.1182013\ttotal: 305ms\tremaining: 271ms\n",
      "53:\tlearn: 0.1165793\ttotal: 312ms\tremaining: 266ms\n",
      "54:\tlearn: 0.1147426\ttotal: 318ms\tremaining: 260ms\n",
      "55:\tlearn: 0.1136771\ttotal: 324ms\tremaining: 254ms\n",
      "56:\tlearn: 0.1113634\ttotal: 330ms\tremaining: 249ms\n",
      "57:\tlearn: 0.1098976\ttotal: 336ms\tremaining: 243ms\n",
      "58:\tlearn: 0.1085510\ttotal: 342ms\tremaining: 238ms\n",
      "59:\tlearn: 0.1070081\ttotal: 347ms\tremaining: 232ms\n",
      "60:\tlearn: 0.1050810\ttotal: 353ms\tremaining: 226ms\n",
      "61:\tlearn: 0.1039554\ttotal: 358ms\tremaining: 219ms\n",
      "62:\tlearn: 0.1029655\ttotal: 363ms\tremaining: 213ms\n",
      "63:\tlearn: 0.1018188\ttotal: 369ms\tremaining: 208ms\n",
      "64:\tlearn: 0.1002720\ttotal: 375ms\tremaining: 202ms\n",
      "65:\tlearn: 0.0992882\ttotal: 381ms\tremaining: 196ms\n",
      "66:\tlearn: 0.0984035\ttotal: 387ms\tremaining: 190ms\n",
      "67:\tlearn: 0.0972013\ttotal: 391ms\tremaining: 184ms\n",
      "68:\tlearn: 0.0961798\ttotal: 397ms\tremaining: 178ms\n",
      "69:\tlearn: 0.0950452\ttotal: 402ms\tremaining: 172ms\n",
      "70:\tlearn: 0.0942115\ttotal: 408ms\tremaining: 166ms\n",
      "71:\tlearn: 0.0934206\ttotal: 413ms\tremaining: 161ms\n",
      "72:\tlearn: 0.0927080\ttotal: 418ms\tremaining: 155ms\n",
      "73:\tlearn: 0.0921909\ttotal: 423ms\tremaining: 149ms\n",
      "74:\tlearn: 0.0910824\ttotal: 428ms\tremaining: 143ms\n",
      "75:\tlearn: 0.0899395\ttotal: 433ms\tremaining: 137ms\n",
      "76:\tlearn: 0.0892967\ttotal: 438ms\tremaining: 131ms\n",
      "77:\tlearn: 0.0883792\ttotal: 443ms\tremaining: 125ms\n",
      "78:\tlearn: 0.0876718\ttotal: 448ms\tremaining: 119ms\n",
      "79:\tlearn: 0.0867184\ttotal: 453ms\tremaining: 113ms\n",
      "80:\tlearn: 0.0861399\ttotal: 458ms\tremaining: 107ms\n",
      "81:\tlearn: 0.0852339\ttotal: 463ms\tremaining: 102ms\n",
      "82:\tlearn: 0.0837381\ttotal: 468ms\tremaining: 95.9ms\n",
      "83:\tlearn: 0.0828339\ttotal: 473ms\tremaining: 90.1ms\n",
      "84:\tlearn: 0.0821235\ttotal: 478ms\tremaining: 84.3ms\n",
      "85:\tlearn: 0.0814570\ttotal: 484ms\tremaining: 78.7ms\n",
      "86:\tlearn: 0.0806323\ttotal: 488ms\tremaining: 73ms\n",
      "87:\tlearn: 0.0799437\ttotal: 494ms\tremaining: 67.3ms\n",
      "88:\tlearn: 0.0793146\ttotal: 498ms\tremaining: 61.6ms\n",
      "89:\tlearn: 0.0784699\ttotal: 503ms\tremaining: 55.9ms\n",
      "90:\tlearn: 0.0775958\ttotal: 509ms\tremaining: 50.3ms\n",
      "91:\tlearn: 0.0768854\ttotal: 514ms\tremaining: 44.7ms\n",
      "92:\tlearn: 0.0765003\ttotal: 519ms\tremaining: 39.1ms\n",
      "93:\tlearn: 0.0756718\ttotal: 524ms\tremaining: 33.4ms\n",
      "94:\tlearn: 0.0750206\ttotal: 528ms\tremaining: 27.8ms\n",
      "95:\tlearn: 0.0746352\ttotal: 534ms\tremaining: 22.3ms\n",
      "96:\tlearn: 0.0738521\ttotal: 539ms\tremaining: 16.7ms\n",
      "97:\tlearn: 0.0734296\ttotal: 544ms\tremaining: 11.1ms\n",
      "98:\tlearn: 0.0727779\ttotal: 549ms\tremaining: 5.55ms\n",
      "99:\tlearn: 0.0724622\ttotal: 554ms\tremaining: 0us\n",
      "0:\tlearn: 1.3366185\ttotal: 7.09ms\tremaining: 702ms\n",
      "1:\tlearn: 1.1489614\ttotal: 13.3ms\tremaining: 650ms\n",
      "2:\tlearn: 1.0053187\ttotal: 20.4ms\tremaining: 661ms\n",
      "3:\tlearn: 0.8953180\ttotal: 27ms\tremaining: 648ms\n",
      "4:\tlearn: 0.8093531\ttotal: 32.7ms\tremaining: 620ms\n",
      "5:\tlearn: 0.7366928\ttotal: 39.6ms\tremaining: 621ms\n",
      "6:\tlearn: 0.6686374\ttotal: 46.5ms\tremaining: 618ms\n",
      "7:\tlearn: 0.6152986\ttotal: 52.2ms\tremaining: 601ms\n",
      "8:\tlearn: 0.5682353\ttotal: 58.1ms\tremaining: 588ms\n",
      "9:\tlearn: 0.5198060\ttotal: 64.5ms\tremaining: 580ms\n",
      "10:\tlearn: 0.4815072\ttotal: 70.2ms\tremaining: 568ms\n",
      "11:\tlearn: 0.4487555\ttotal: 75.4ms\tremaining: 553ms\n",
      "12:\tlearn: 0.4185226\ttotal: 80.5ms\tremaining: 539ms\n",
      "13:\tlearn: 0.3933454\ttotal: 85.5ms\tremaining: 525ms\n",
      "14:\tlearn: 0.3706571\ttotal: 90.4ms\tremaining: 512ms\n",
      "15:\tlearn: 0.3491939\ttotal: 97.1ms\tremaining: 510ms\n",
      "16:\tlearn: 0.3290380\ttotal: 103ms\tremaining: 504ms\n",
      "17:\tlearn: 0.3094743\ttotal: 109ms\tremaining: 497ms\n",
      "18:\tlearn: 0.2938610\ttotal: 115ms\tremaining: 490ms\n",
      "19:\tlearn: 0.2815324\ttotal: 121ms\tremaining: 482ms\n",
      "20:\tlearn: 0.2674508\ttotal: 126ms\tremaining: 476ms\n",
      "21:\tlearn: 0.2530842\ttotal: 132ms\tremaining: 468ms\n",
      "22:\tlearn: 0.2412084\ttotal: 138ms\tremaining: 463ms\n",
      "23:\tlearn: 0.2311567\ttotal: 145ms\tremaining: 458ms\n",
      "24:\tlearn: 0.2216495\ttotal: 151ms\tremaining: 452ms\n",
      "25:\tlearn: 0.2135722\ttotal: 157ms\tremaining: 446ms\n",
      "26:\tlearn: 0.2059383\ttotal: 162ms\tremaining: 439ms\n",
      "27:\tlearn: 0.1985832\ttotal: 168ms\tremaining: 433ms\n",
      "28:\tlearn: 0.1906933\ttotal: 174ms\tremaining: 425ms\n",
      "29:\tlearn: 0.1845961\ttotal: 179ms\tremaining: 417ms\n",
      "30:\tlearn: 0.1782160\ttotal: 184ms\tremaining: 410ms\n",
      "31:\tlearn: 0.1742173\ttotal: 190ms\tremaining: 404ms\n",
      "32:\tlearn: 0.1698116\ttotal: 195ms\tremaining: 396ms\n",
      "33:\tlearn: 0.1657310\ttotal: 201ms\tremaining: 390ms\n",
      "34:\tlearn: 0.1608660\ttotal: 206ms\tremaining: 382ms\n",
      "35:\tlearn: 0.1570855\ttotal: 211ms\tremaining: 375ms\n",
      "36:\tlearn: 0.1514623\ttotal: 216ms\tremaining: 367ms\n",
      "37:\tlearn: 0.1478054\ttotal: 221ms\tremaining: 361ms\n",
      "38:\tlearn: 0.1440010\ttotal: 227ms\tremaining: 354ms\n",
      "39:\tlearn: 0.1404230\ttotal: 232ms\tremaining: 347ms\n",
      "40:\tlearn: 0.1358894\ttotal: 236ms\tremaining: 340ms\n",
      "41:\tlearn: 0.1336024\ttotal: 242ms\tremaining: 334ms\n",
      "42:\tlearn: 0.1307811\ttotal: 246ms\tremaining: 327ms\n",
      "43:\tlearn: 0.1293443\ttotal: 251ms\tremaining: 320ms\n",
      "44:\tlearn: 0.1263315\ttotal: 256ms\tremaining: 313ms\n",
      "45:\tlearn: 0.1238985\ttotal: 262ms\tremaining: 307ms\n",
      "46:\tlearn: 0.1221462\ttotal: 267ms\tremaining: 301ms\n",
      "47:\tlearn: 0.1201230\ttotal: 272ms\tremaining: 295ms\n",
      "48:\tlearn: 0.1183652\ttotal: 277ms\tremaining: 288ms\n",
      "49:\tlearn: 0.1165999\ttotal: 282ms\tremaining: 282ms\n",
      "50:\tlearn: 0.1149404\ttotal: 287ms\tremaining: 276ms\n",
      "51:\tlearn: 0.1138469\ttotal: 292ms\tremaining: 269ms\n",
      "52:\tlearn: 0.1122288\ttotal: 297ms\tremaining: 263ms\n",
      "53:\tlearn: 0.1106749\ttotal: 302ms\tremaining: 257ms\n",
      "54:\tlearn: 0.1082260\ttotal: 308ms\tremaining: 252ms\n",
      "55:\tlearn: 0.1072496\ttotal: 312ms\tremaining: 245ms\n",
      "56:\tlearn: 0.1048805\ttotal: 317ms\tremaining: 239ms\n",
      "57:\tlearn: 0.1029372\ttotal: 323ms\tremaining: 234ms\n",
      "58:\tlearn: 0.1015274\ttotal: 329ms\tremaining: 229ms\n",
      "59:\tlearn: 0.0997993\ttotal: 335ms\tremaining: 223ms\n",
      "60:\tlearn: 0.0981517\ttotal: 341ms\tremaining: 218ms\n",
      "61:\tlearn: 0.0970838\ttotal: 345ms\tremaining: 212ms\n",
      "62:\tlearn: 0.0958603\ttotal: 351ms\tremaining: 206ms\n",
      "63:\tlearn: 0.0948193\ttotal: 357ms\tremaining: 201ms\n",
      "64:\tlearn: 0.0940833\ttotal: 361ms\tremaining: 195ms\n",
      "65:\tlearn: 0.0928339\ttotal: 366ms\tremaining: 189ms\n",
      "66:\tlearn: 0.0913352\ttotal: 372ms\tremaining: 183ms\n",
      "67:\tlearn: 0.0907165\ttotal: 377ms\tremaining: 177ms\n",
      "68:\tlearn: 0.0891477\ttotal: 383ms\tremaining: 172ms\n",
      "69:\tlearn: 0.0881797\ttotal: 387ms\tremaining: 166ms\n",
      "70:\tlearn: 0.0874757\ttotal: 392ms\tremaining: 160ms\n",
      "71:\tlearn: 0.0865009\ttotal: 398ms\tremaining: 155ms\n",
      "72:\tlearn: 0.0858887\ttotal: 403ms\tremaining: 149ms\n",
      "73:\tlearn: 0.0849559\ttotal: 408ms\tremaining: 143ms\n",
      "74:\tlearn: 0.0842454\ttotal: 414ms\tremaining: 138ms\n",
      "75:\tlearn: 0.0832461\ttotal: 420ms\tremaining: 133ms\n",
      "76:\tlearn: 0.0827439\ttotal: 425ms\tremaining: 127ms\n",
      "77:\tlearn: 0.0819646\ttotal: 430ms\tremaining: 121ms\n",
      "78:\tlearn: 0.0811315\ttotal: 435ms\tremaining: 116ms\n",
      "79:\tlearn: 0.0802525\ttotal: 439ms\tremaining: 110ms\n",
      "80:\tlearn: 0.0796070\ttotal: 444ms\tremaining: 104ms\n",
      "81:\tlearn: 0.0788240\ttotal: 449ms\tremaining: 98.5ms\n",
      "82:\tlearn: 0.0783274\ttotal: 453ms\tremaining: 92.9ms\n",
      "83:\tlearn: 0.0771233\ttotal: 458ms\tremaining: 87.3ms\n",
      "84:\tlearn: 0.0765721\ttotal: 463ms\tremaining: 81.7ms\n",
      "85:\tlearn: 0.0757317\ttotal: 469ms\tremaining: 76.3ms\n",
      "86:\tlearn: 0.0749472\ttotal: 474ms\tremaining: 70.8ms\n",
      "87:\tlearn: 0.0743601\ttotal: 479ms\tremaining: 65.3ms\n",
      "88:\tlearn: 0.0738924\ttotal: 484ms\tremaining: 59.8ms\n",
      "89:\tlearn: 0.0734715\ttotal: 490ms\tremaining: 54.5ms\n",
      "90:\tlearn: 0.0730793\ttotal: 496ms\tremaining: 49.1ms\n",
      "91:\tlearn: 0.0725461\ttotal: 503ms\tremaining: 43.7ms\n",
      "92:\tlearn: 0.0718353\ttotal: 509ms\tremaining: 38.3ms\n",
      "93:\tlearn: 0.0710014\ttotal: 515ms\tremaining: 32.8ms\n",
      "94:\tlearn: 0.0705450\ttotal: 521ms\tremaining: 27.4ms\n",
      "95:\tlearn: 0.0700889\ttotal: 527ms\tremaining: 22ms\n",
      "96:\tlearn: 0.0698293\ttotal: 532ms\tremaining: 16.5ms\n",
      "97:\tlearn: 0.0692936\ttotal: 538ms\tremaining: 11ms\n",
      "98:\tlearn: 0.0685551\ttotal: 543ms\tremaining: 5.49ms\n",
      "99:\tlearn: 0.0679734\ttotal: 549ms\tremaining: 0us\n",
      "0:\tlearn: 1.3428846\ttotal: 8.34ms\tremaining: 826ms\n",
      "1:\tlearn: 1.1554105\ttotal: 15.3ms\tremaining: 751ms\n",
      "2:\tlearn: 1.0125000\ttotal: 22.5ms\tremaining: 727ms\n",
      "3:\tlearn: 0.8993795\ttotal: 29.1ms\tremaining: 698ms\n",
      "4:\tlearn: 0.8162347\ttotal: 35.6ms\tremaining: 677ms\n",
      "5:\tlearn: 0.7468142\ttotal: 41.8ms\tremaining: 655ms\n",
      "6:\tlearn: 0.6861277\ttotal: 49.1ms\tremaining: 652ms\n",
      "7:\tlearn: 0.6330991\ttotal: 55.9ms\tremaining: 643ms\n",
      "8:\tlearn: 0.5869983\ttotal: 61.7ms\tremaining: 624ms\n",
      "9:\tlearn: 0.5413992\ttotal: 67.1ms\tremaining: 604ms\n",
      "10:\tlearn: 0.5068085\ttotal: 73.1ms\tremaining: 591ms\n",
      "11:\tlearn: 0.4708932\ttotal: 78.4ms\tremaining: 575ms\n",
      "12:\tlearn: 0.4416567\ttotal: 83.4ms\tremaining: 558ms\n",
      "13:\tlearn: 0.4139371\ttotal: 89ms\tremaining: 547ms\n",
      "14:\tlearn: 0.3910055\ttotal: 95.2ms\tremaining: 539ms\n",
      "15:\tlearn: 0.3679629\ttotal: 100ms\tremaining: 528ms\n",
      "16:\tlearn: 0.3455367\ttotal: 106ms\tremaining: 516ms\n",
      "17:\tlearn: 0.3277082\ttotal: 112ms\tremaining: 509ms\n",
      "18:\tlearn: 0.3125739\ttotal: 118ms\tremaining: 502ms\n",
      "19:\tlearn: 0.2976880\ttotal: 123ms\tremaining: 491ms\n",
      "20:\tlearn: 0.2834473\ttotal: 128ms\tremaining: 481ms\n",
      "21:\tlearn: 0.2695296\ttotal: 133ms\tremaining: 471ms\n",
      "22:\tlearn: 0.2577622\ttotal: 138ms\tremaining: 462ms\n",
      "23:\tlearn: 0.2479630\ttotal: 144ms\tremaining: 455ms\n",
      "24:\tlearn: 0.2380722\ttotal: 148ms\tremaining: 445ms\n",
      "25:\tlearn: 0.2291784\ttotal: 153ms\tremaining: 436ms\n",
      "26:\tlearn: 0.2206744\ttotal: 158ms\tremaining: 427ms\n",
      "27:\tlearn: 0.2128866\ttotal: 164ms\tremaining: 420ms\n",
      "28:\tlearn: 0.2058515\ttotal: 168ms\tremaining: 412ms\n",
      "29:\tlearn: 0.1993344\ttotal: 173ms\tremaining: 405ms\n",
      "30:\tlearn: 0.1928308\ttotal: 178ms\tremaining: 397ms\n",
      "31:\tlearn: 0.1880190\ttotal: 183ms\tremaining: 388ms\n",
      "32:\tlearn: 0.1829755\ttotal: 187ms\tremaining: 380ms\n",
      "33:\tlearn: 0.1783535\ttotal: 192ms\tremaining: 372ms\n",
      "34:\tlearn: 0.1740049\ttotal: 196ms\tremaining: 365ms\n",
      "35:\tlearn: 0.1691473\ttotal: 201ms\tremaining: 358ms\n",
      "36:\tlearn: 0.1647200\ttotal: 206ms\tremaining: 351ms\n",
      "37:\tlearn: 0.1611518\ttotal: 211ms\tremaining: 345ms\n",
      "38:\tlearn: 0.1569738\ttotal: 216ms\tremaining: 337ms\n",
      "39:\tlearn: 0.1529582\ttotal: 220ms\tremaining: 331ms\n",
      "40:\tlearn: 0.1499525\ttotal: 225ms\tremaining: 323ms\n",
      "41:\tlearn: 0.1474496\ttotal: 229ms\tremaining: 317ms\n",
      "42:\tlearn: 0.1444136\ttotal: 234ms\tremaining: 310ms\n",
      "43:\tlearn: 0.1418856\ttotal: 238ms\tremaining: 303ms\n",
      "44:\tlearn: 0.1381110\ttotal: 243ms\tremaining: 297ms\n",
      "45:\tlearn: 0.1355365\ttotal: 250ms\tremaining: 293ms\n",
      "46:\tlearn: 0.1323494\ttotal: 255ms\tremaining: 288ms\n",
      "47:\tlearn: 0.1290575\ttotal: 261ms\tremaining: 283ms\n",
      "48:\tlearn: 0.1272805\ttotal: 266ms\tremaining: 276ms\n",
      "49:\tlearn: 0.1254700\ttotal: 271ms\tremaining: 271ms\n",
      "50:\tlearn: 0.1242068\ttotal: 276ms\tremaining: 266ms\n",
      "51:\tlearn: 0.1227941\ttotal: 281ms\tremaining: 259ms\n",
      "52:\tlearn: 0.1210133\ttotal: 286ms\tremaining: 254ms\n",
      "53:\tlearn: 0.1198027\ttotal: 291ms\tremaining: 248ms\n",
      "54:\tlearn: 0.1171461\ttotal: 297ms\tremaining: 243ms\n",
      "55:\tlearn: 0.1157550\ttotal: 302ms\tremaining: 237ms\n",
      "56:\tlearn: 0.1135781\ttotal: 308ms\tremaining: 232ms\n",
      "57:\tlearn: 0.1123406\ttotal: 313ms\tremaining: 226ms\n",
      "58:\tlearn: 0.1110249\ttotal: 317ms\tremaining: 220ms\n",
      "59:\tlearn: 0.1098255\ttotal: 322ms\tremaining: 215ms\n",
      "60:\tlearn: 0.1078794\ttotal: 327ms\tremaining: 209ms\n",
      "61:\tlearn: 0.1061959\ttotal: 333ms\tremaining: 204ms\n",
      "62:\tlearn: 0.1049427\ttotal: 337ms\tremaining: 198ms\n",
      "63:\tlearn: 0.1038477\ttotal: 343ms\tremaining: 193ms\n",
      "64:\tlearn: 0.1026410\ttotal: 348ms\tremaining: 188ms\n",
      "65:\tlearn: 0.1011819\ttotal: 353ms\tremaining: 182ms\n",
      "66:\tlearn: 0.0996587\ttotal: 357ms\tremaining: 176ms\n",
      "67:\tlearn: 0.0983598\ttotal: 362ms\tremaining: 170ms\n",
      "68:\tlearn: 0.0975090\ttotal: 367ms\tremaining: 165ms\n",
      "69:\tlearn: 0.0965672\ttotal: 372ms\tremaining: 159ms\n",
      "70:\tlearn: 0.0957167\ttotal: 377ms\tremaining: 154ms\n",
      "71:\tlearn: 0.0946420\ttotal: 383ms\tremaining: 149ms\n",
      "72:\tlearn: 0.0937064\ttotal: 390ms\tremaining: 144ms\n",
      "73:\tlearn: 0.0929568\ttotal: 396ms\tremaining: 139ms\n",
      "74:\tlearn: 0.0916637\ttotal: 402ms\tremaining: 134ms\n",
      "75:\tlearn: 0.0904895\ttotal: 408ms\tremaining: 129ms\n",
      "76:\tlearn: 0.0895104\ttotal: 415ms\tremaining: 124ms\n",
      "77:\tlearn: 0.0888397\ttotal: 421ms\tremaining: 119ms\n",
      "78:\tlearn: 0.0882232\ttotal: 428ms\tremaining: 114ms\n",
      "79:\tlearn: 0.0873129\ttotal: 434ms\tremaining: 108ms\n",
      "80:\tlearn: 0.0865138\ttotal: 440ms\tremaining: 103ms\n",
      "81:\tlearn: 0.0857284\ttotal: 446ms\tremaining: 97.8ms\n",
      "82:\tlearn: 0.0845817\ttotal: 451ms\tremaining: 92.4ms\n",
      "83:\tlearn: 0.0833405\ttotal: 458ms\tremaining: 87.2ms\n",
      "84:\tlearn: 0.0824591\ttotal: 464ms\tremaining: 81.9ms\n",
      "85:\tlearn: 0.0818346\ttotal: 470ms\tremaining: 76.6ms\n",
      "86:\tlearn: 0.0812997\ttotal: 478ms\tremaining: 71.4ms\n",
      "87:\tlearn: 0.0803596\ttotal: 485ms\tremaining: 66.2ms\n",
      "88:\tlearn: 0.0799204\ttotal: 492ms\tremaining: 60.8ms\n",
      "89:\tlearn: 0.0793370\ttotal: 499ms\tremaining: 55.4ms\n",
      "90:\tlearn: 0.0784296\ttotal: 504ms\tremaining: 49.9ms\n",
      "91:\tlearn: 0.0770052\ttotal: 511ms\tremaining: 44.4ms\n",
      "92:\tlearn: 0.0762449\ttotal: 516ms\tremaining: 38.8ms\n",
      "93:\tlearn: 0.0756264\ttotal: 521ms\tremaining: 33.3ms\n",
      "94:\tlearn: 0.0749736\ttotal: 527ms\tremaining: 27.8ms\n",
      "95:\tlearn: 0.0744933\ttotal: 533ms\tremaining: 22.2ms\n",
      "96:\tlearn: 0.0739009\ttotal: 540ms\tremaining: 16.7ms\n",
      "97:\tlearn: 0.0735728\ttotal: 547ms\tremaining: 11.2ms\n",
      "98:\tlearn: 0.0730453\ttotal: 553ms\tremaining: 5.58ms\n",
      "99:\tlearn: 0.0723588\ttotal: 559ms\tremaining: 0us\n",
      "0:\tlearn: 1.3414688\ttotal: 7.13ms\tremaining: 705ms\n",
      "1:\tlearn: 1.1561902\ttotal: 14ms\tremaining: 687ms\n",
      "2:\tlearn: 1.0126082\ttotal: 20.5ms\tremaining: 664ms\n",
      "3:\tlearn: 0.9011587\ttotal: 26.4ms\tremaining: 634ms\n",
      "4:\tlearn: 0.8152606\ttotal: 32.4ms\tremaining: 616ms\n",
      "5:\tlearn: 0.7462258\ttotal: 37.6ms\tremaining: 589ms\n",
      "6:\tlearn: 0.6785406\ttotal: 44.1ms\tremaining: 586ms\n",
      "7:\tlearn: 0.6239297\ttotal: 49.2ms\tremaining: 565ms\n",
      "8:\tlearn: 0.5770788\ttotal: 54.4ms\tremaining: 550ms\n",
      "9:\tlearn: 0.5284749\ttotal: 60.3ms\tremaining: 542ms\n",
      "10:\tlearn: 0.4891979\ttotal: 66.8ms\tremaining: 541ms\n",
      "11:\tlearn: 0.4557628\ttotal: 72.6ms\tremaining: 533ms\n",
      "12:\tlearn: 0.4274597\ttotal: 78.2ms\tremaining: 524ms\n",
      "13:\tlearn: 0.4020666\ttotal: 83.4ms\tremaining: 512ms\n",
      "14:\tlearn: 0.3774437\ttotal: 88.9ms\tremaining: 504ms\n",
      "15:\tlearn: 0.3562817\ttotal: 94.9ms\tremaining: 498ms\n",
      "16:\tlearn: 0.3354976\ttotal: 99.8ms\tremaining: 487ms\n",
      "17:\tlearn: 0.3186221\ttotal: 106ms\tremaining: 482ms\n",
      "18:\tlearn: 0.3043051\ttotal: 111ms\tremaining: 474ms\n",
      "19:\tlearn: 0.2888698\ttotal: 117ms\tremaining: 468ms\n",
      "20:\tlearn: 0.2762769\ttotal: 123ms\tremaining: 462ms\n",
      "21:\tlearn: 0.2660338\ttotal: 129ms\tremaining: 457ms\n",
      "22:\tlearn: 0.2542472\ttotal: 135ms\tremaining: 453ms\n",
      "23:\tlearn: 0.2442045\ttotal: 141ms\tremaining: 447ms\n",
      "24:\tlearn: 0.2347765\ttotal: 147ms\tremaining: 441ms\n",
      "25:\tlearn: 0.2267112\ttotal: 153ms\tremaining: 436ms\n",
      "26:\tlearn: 0.2180885\ttotal: 158ms\tremaining: 428ms\n",
      "27:\tlearn: 0.2111513\ttotal: 163ms\tremaining: 419ms\n",
      "28:\tlearn: 0.2031124\ttotal: 168ms\tremaining: 411ms\n",
      "29:\tlearn: 0.1941855\ttotal: 173ms\tremaining: 403ms\n",
      "30:\tlearn: 0.1887709\ttotal: 178ms\tremaining: 396ms\n",
      "31:\tlearn: 0.1836748\ttotal: 184ms\tremaining: 390ms\n",
      "32:\tlearn: 0.1778166\ttotal: 190ms\tremaining: 385ms\n",
      "33:\tlearn: 0.1729417\ttotal: 195ms\tremaining: 378ms\n",
      "34:\tlearn: 0.1673432\ttotal: 200ms\tremaining: 371ms\n",
      "35:\tlearn: 0.1632541\ttotal: 205ms\tremaining: 365ms\n",
      "36:\tlearn: 0.1587588\ttotal: 211ms\tremaining: 359ms\n",
      "37:\tlearn: 0.1542734\ttotal: 216ms\tremaining: 353ms\n",
      "38:\tlearn: 0.1506203\ttotal: 222ms\tremaining: 347ms\n",
      "39:\tlearn: 0.1473907\ttotal: 228ms\tremaining: 342ms\n",
      "40:\tlearn: 0.1436658\ttotal: 234ms\tremaining: 336ms\n",
      "41:\tlearn: 0.1408057\ttotal: 239ms\tremaining: 330ms\n",
      "42:\tlearn: 0.1372899\ttotal: 244ms\tremaining: 323ms\n",
      "43:\tlearn: 0.1352764\ttotal: 248ms\tremaining: 316ms\n",
      "44:\tlearn: 0.1326596\ttotal: 254ms\tremaining: 310ms\n",
      "45:\tlearn: 0.1305049\ttotal: 259ms\tremaining: 303ms\n",
      "46:\tlearn: 0.1283527\ttotal: 264ms\tremaining: 297ms\n",
      "47:\tlearn: 0.1254850\ttotal: 268ms\tremaining: 291ms\n",
      "48:\tlearn: 0.1232703\ttotal: 274ms\tremaining: 285ms\n",
      "49:\tlearn: 0.1213047\ttotal: 278ms\tremaining: 278ms\n",
      "50:\tlearn: 0.1193757\ttotal: 284ms\tremaining: 273ms\n",
      "51:\tlearn: 0.1177405\ttotal: 289ms\tremaining: 267ms\n",
      "52:\tlearn: 0.1156540\ttotal: 295ms\tremaining: 261ms\n",
      "53:\tlearn: 0.1139409\ttotal: 299ms\tremaining: 255ms\n",
      "54:\tlearn: 0.1116214\ttotal: 304ms\tremaining: 249ms\n",
      "55:\tlearn: 0.1103928\ttotal: 310ms\tremaining: 243ms\n",
      "56:\tlearn: 0.1083485\ttotal: 315ms\tremaining: 237ms\n",
      "57:\tlearn: 0.1072665\ttotal: 320ms\tremaining: 231ms\n",
      "58:\tlearn: 0.1060609\ttotal: 324ms\tremaining: 225ms\n",
      "59:\tlearn: 0.1043763\ttotal: 329ms\tremaining: 220ms\n",
      "60:\tlearn: 0.1029636\ttotal: 335ms\tremaining: 214ms\n",
      "61:\tlearn: 0.1015257\ttotal: 340ms\tremaining: 208ms\n",
      "62:\tlearn: 0.1005146\ttotal: 345ms\tremaining: 203ms\n",
      "63:\tlearn: 0.0993438\ttotal: 350ms\tremaining: 197ms\n",
      "64:\tlearn: 0.0980785\ttotal: 355ms\tremaining: 191ms\n",
      "65:\tlearn: 0.0972105\ttotal: 361ms\tremaining: 186ms\n",
      "66:\tlearn: 0.0960973\ttotal: 366ms\tremaining: 180ms\n",
      "67:\tlearn: 0.0949522\ttotal: 371ms\tremaining: 174ms\n",
      "68:\tlearn: 0.0935773\ttotal: 377ms\tremaining: 169ms\n",
      "69:\tlearn: 0.0926073\ttotal: 383ms\tremaining: 164ms\n",
      "70:\tlearn: 0.0916983\ttotal: 389ms\tremaining: 159ms\n",
      "71:\tlearn: 0.0908138\ttotal: 396ms\tremaining: 154ms\n",
      "72:\tlearn: 0.0893898\ttotal: 402ms\tremaining: 149ms\n",
      "73:\tlearn: 0.0886597\ttotal: 408ms\tremaining: 143ms\n",
      "74:\tlearn: 0.0877631\ttotal: 415ms\tremaining: 138ms\n",
      "75:\tlearn: 0.0868821\ttotal: 421ms\tremaining: 133ms\n",
      "76:\tlearn: 0.0860640\ttotal: 427ms\tremaining: 128ms\n",
      "77:\tlearn: 0.0851707\ttotal: 433ms\tremaining: 122ms\n",
      "78:\tlearn: 0.0845029\ttotal: 440ms\tremaining: 117ms\n",
      "79:\tlearn: 0.0835707\ttotal: 445ms\tremaining: 111ms\n",
      "80:\tlearn: 0.0828003\ttotal: 450ms\tremaining: 106ms\n",
      "81:\tlearn: 0.0819083\ttotal: 455ms\tremaining: 100ms\n",
      "82:\tlearn: 0.0810426\ttotal: 461ms\tremaining: 94.4ms\n",
      "83:\tlearn: 0.0802198\ttotal: 466ms\tremaining: 88.8ms\n",
      "84:\tlearn: 0.0797187\ttotal: 471ms\tremaining: 83.1ms\n",
      "85:\tlearn: 0.0787176\ttotal: 476ms\tremaining: 77.5ms\n",
      "86:\tlearn: 0.0780014\ttotal: 480ms\tremaining: 71.8ms\n",
      "87:\tlearn: 0.0774743\ttotal: 487ms\tremaining: 66.4ms\n",
      "88:\tlearn: 0.0769541\ttotal: 493ms\tremaining: 60.9ms\n",
      "89:\tlearn: 0.0760653\ttotal: 499ms\tremaining: 55.4ms\n",
      "90:\tlearn: 0.0754390\ttotal: 505ms\tremaining: 49.9ms\n",
      "91:\tlearn: 0.0748865\ttotal: 511ms\tremaining: 44.4ms\n",
      "92:\tlearn: 0.0745905\ttotal: 517ms\tremaining: 38.9ms\n",
      "93:\tlearn: 0.0740886\ttotal: 522ms\tremaining: 33.3ms\n",
      "94:\tlearn: 0.0735414\ttotal: 528ms\tremaining: 27.8ms\n",
      "95:\tlearn: 0.0731150\ttotal: 534ms\tremaining: 22.2ms\n",
      "96:\tlearn: 0.0723429\ttotal: 540ms\tremaining: 16.7ms\n",
      "97:\tlearn: 0.0719554\ttotal: 546ms\tremaining: 11.2ms\n",
      "98:\tlearn: 0.0714891\ttotal: 553ms\tremaining: 5.58ms\n",
      "99:\tlearn: 0.0713297\ttotal: 558ms\tremaining: 0us\n",
      "0:\tlearn: 1.3311912\ttotal: 7ms\tremaining: 693ms\n",
      "1:\tlearn: 1.1438355\ttotal: 14ms\tremaining: 687ms\n",
      "2:\tlearn: 1.0050694\ttotal: 20.9ms\tremaining: 674ms\n",
      "3:\tlearn: 0.8949487\ttotal: 27.3ms\tremaining: 655ms\n",
      "4:\tlearn: 0.8062765\ttotal: 34.2ms\tremaining: 650ms\n",
      "5:\tlearn: 0.7363321\ttotal: 40.9ms\tremaining: 641ms\n",
      "6:\tlearn: 0.6732625\ttotal: 48ms\tremaining: 638ms\n",
      "7:\tlearn: 0.6177145\ttotal: 54.6ms\tremaining: 628ms\n",
      "8:\tlearn: 0.5683037\ttotal: 60.9ms\tremaining: 616ms\n",
      "9:\tlearn: 0.5268492\ttotal: 67.5ms\tremaining: 608ms\n",
      "10:\tlearn: 0.4871600\ttotal: 75.2ms\tremaining: 608ms\n",
      "11:\tlearn: 0.4537910\ttotal: 81.6ms\tremaining: 598ms\n",
      "12:\tlearn: 0.4217282\ttotal: 87.7ms\tremaining: 587ms\n",
      "13:\tlearn: 0.3948655\ttotal: 93.7ms\tremaining: 576ms\n",
      "14:\tlearn: 0.3714223\ttotal: 99.8ms\tremaining: 565ms\n",
      "15:\tlearn: 0.3510235\ttotal: 106ms\tremaining: 556ms\n",
      "16:\tlearn: 0.3304109\ttotal: 111ms\tremaining: 544ms\n",
      "17:\tlearn: 0.3106361\ttotal: 117ms\tremaining: 532ms\n",
      "18:\tlearn: 0.2944528\ttotal: 122ms\tremaining: 521ms\n",
      "19:\tlearn: 0.2809476\ttotal: 127ms\tremaining: 510ms\n",
      "20:\tlearn: 0.2683199\ttotal: 133ms\tremaining: 502ms\n",
      "21:\tlearn: 0.2558859\ttotal: 138ms\tremaining: 490ms\n",
      "22:\tlearn: 0.2437700\ttotal: 144ms\tremaining: 482ms\n",
      "23:\tlearn: 0.2334571\ttotal: 150ms\tremaining: 475ms\n",
      "24:\tlearn: 0.2240932\ttotal: 155ms\tremaining: 464ms\n",
      "25:\tlearn: 0.2161211\ttotal: 160ms\tremaining: 454ms\n",
      "26:\tlearn: 0.2076213\ttotal: 165ms\tremaining: 445ms\n",
      "27:\tlearn: 0.1995130\ttotal: 170ms\tremaining: 437ms\n",
      "28:\tlearn: 0.1912655\ttotal: 175ms\tremaining: 429ms\n",
      "29:\tlearn: 0.1831952\ttotal: 180ms\tremaining: 420ms\n",
      "30:\tlearn: 0.1765772\ttotal: 185ms\tremaining: 412ms\n",
      "31:\tlearn: 0.1718896\ttotal: 190ms\tremaining: 404ms\n",
      "32:\tlearn: 0.1665998\ttotal: 195ms\tremaining: 396ms\n",
      "33:\tlearn: 0.1623496\ttotal: 200ms\tremaining: 387ms\n",
      "34:\tlearn: 0.1580465\ttotal: 204ms\tremaining: 380ms\n",
      "35:\tlearn: 0.1540127\ttotal: 210ms\tremaining: 374ms\n",
      "36:\tlearn: 0.1499015\ttotal: 216ms\tremaining: 369ms\n",
      "37:\tlearn: 0.1458285\ttotal: 222ms\tremaining: 363ms\n",
      "38:\tlearn: 0.1421360\ttotal: 228ms\tremaining: 357ms\n",
      "39:\tlearn: 0.1387696\ttotal: 233ms\tremaining: 350ms\n",
      "40:\tlearn: 0.1364903\ttotal: 239ms\tremaining: 343ms\n",
      "41:\tlearn: 0.1329919\ttotal: 244ms\tremaining: 337ms\n",
      "42:\tlearn: 0.1302739\ttotal: 249ms\tremaining: 330ms\n",
      "43:\tlearn: 0.1278700\ttotal: 254ms\tremaining: 323ms\n",
      "44:\tlearn: 0.1257922\ttotal: 260ms\tremaining: 317ms\n",
      "45:\tlearn: 0.1229029\ttotal: 264ms\tremaining: 310ms\n",
      "46:\tlearn: 0.1200029\ttotal: 270ms\tremaining: 305ms\n",
      "47:\tlearn: 0.1176877\ttotal: 275ms\tremaining: 298ms\n",
      "48:\tlearn: 0.1162207\ttotal: 280ms\tremaining: 291ms\n",
      "49:\tlearn: 0.1145182\ttotal: 285ms\tremaining: 285ms\n",
      "50:\tlearn: 0.1124398\ttotal: 291ms\tremaining: 279ms\n",
      "51:\tlearn: 0.1107597\ttotal: 296ms\tremaining: 273ms\n",
      "52:\tlearn: 0.1093179\ttotal: 300ms\tremaining: 266ms\n",
      "53:\tlearn: 0.1077531\ttotal: 305ms\tremaining: 260ms\n",
      "54:\tlearn: 0.1059844\ttotal: 310ms\tremaining: 254ms\n",
      "55:\tlearn: 0.1042107\ttotal: 315ms\tremaining: 247ms\n",
      "56:\tlearn: 0.1022127\ttotal: 320ms\tremaining: 241ms\n",
      "57:\tlearn: 0.1007515\ttotal: 325ms\tremaining: 235ms\n",
      "58:\tlearn: 0.0996001\ttotal: 330ms\tremaining: 229ms\n",
      "59:\tlearn: 0.0981629\ttotal: 334ms\tremaining: 223ms\n",
      "60:\tlearn: 0.0964990\ttotal: 339ms\tremaining: 217ms\n",
      "61:\tlearn: 0.0949928\ttotal: 345ms\tremaining: 211ms\n",
      "62:\tlearn: 0.0940611\ttotal: 350ms\tremaining: 206ms\n",
      "63:\tlearn: 0.0929787\ttotal: 355ms\tremaining: 200ms\n",
      "64:\tlearn: 0.0918873\ttotal: 361ms\tremaining: 194ms\n",
      "65:\tlearn: 0.0904742\ttotal: 366ms\tremaining: 188ms\n",
      "66:\tlearn: 0.0889707\ttotal: 371ms\tremaining: 183ms\n",
      "67:\tlearn: 0.0877280\ttotal: 376ms\tremaining: 177ms\n",
      "68:\tlearn: 0.0867732\ttotal: 381ms\tremaining: 171ms\n",
      "69:\tlearn: 0.0858199\ttotal: 386ms\tremaining: 166ms\n",
      "70:\tlearn: 0.0849923\ttotal: 392ms\tremaining: 160ms\n",
      "71:\tlearn: 0.0842476\ttotal: 397ms\tremaining: 154ms\n",
      "72:\tlearn: 0.0834906\ttotal: 402ms\tremaining: 149ms\n",
      "73:\tlearn: 0.0827031\ttotal: 407ms\tremaining: 143ms\n",
      "74:\tlearn: 0.0820439\ttotal: 413ms\tremaining: 138ms\n",
      "75:\tlearn: 0.0807398\ttotal: 419ms\tremaining: 132ms\n",
      "76:\tlearn: 0.0796992\ttotal: 425ms\tremaining: 127ms\n",
      "77:\tlearn: 0.0783904\ttotal: 430ms\tremaining: 121ms\n",
      "78:\tlearn: 0.0777578\ttotal: 435ms\tremaining: 116ms\n",
      "79:\tlearn: 0.0771736\ttotal: 441ms\tremaining: 110ms\n",
      "80:\tlearn: 0.0765908\ttotal: 446ms\tremaining: 105ms\n",
      "81:\tlearn: 0.0759213\ttotal: 451ms\tremaining: 99.1ms\n",
      "82:\tlearn: 0.0753550\ttotal: 456ms\tremaining: 93.4ms\n",
      "83:\tlearn: 0.0743553\ttotal: 461ms\tremaining: 87.7ms\n",
      "84:\tlearn: 0.0737246\ttotal: 465ms\tremaining: 82.1ms\n",
      "85:\tlearn: 0.0728906\ttotal: 470ms\tremaining: 76.5ms\n",
      "86:\tlearn: 0.0722547\ttotal: 475ms\tremaining: 71ms\n",
      "87:\tlearn: 0.0712245\ttotal: 480ms\tremaining: 65.5ms\n",
      "88:\tlearn: 0.0709220\ttotal: 486ms\tremaining: 60ms\n",
      "89:\tlearn: 0.0699695\ttotal: 491ms\tremaining: 54.6ms\n",
      "90:\tlearn: 0.0695112\ttotal: 496ms\tremaining: 49ms\n",
      "91:\tlearn: 0.0689137\ttotal: 501ms\tremaining: 43.5ms\n",
      "92:\tlearn: 0.0685346\ttotal: 505ms\tremaining: 38ms\n",
      "93:\tlearn: 0.0681152\ttotal: 510ms\tremaining: 32.6ms\n",
      "94:\tlearn: 0.0677232\ttotal: 515ms\tremaining: 27.1ms\n",
      "95:\tlearn: 0.0674719\ttotal: 522ms\tremaining: 21.7ms\n",
      "96:\tlearn: 0.0668467\ttotal: 528ms\tremaining: 16.3ms\n",
      "97:\tlearn: 0.0662088\ttotal: 534ms\tremaining: 10.9ms\n",
      "98:\tlearn: 0.0656170\ttotal: 541ms\tremaining: 5.47ms\n",
      "99:\tlearn: 0.0652163\ttotal: 548ms\tremaining: 0us\n",
      "0:\tlearn: 1.3386012\ttotal: 6.83ms\tremaining: 677ms\n",
      "1:\tlearn: 1.1493885\ttotal: 14.1ms\tremaining: 690ms\n",
      "2:\tlearn: 1.0081120\ttotal: 20.5ms\tremaining: 663ms\n",
      "3:\tlearn: 0.8975649\ttotal: 27.1ms\tremaining: 652ms\n",
      "4:\tlearn: 0.8155639\ttotal: 33.1ms\tremaining: 628ms\n",
      "5:\tlearn: 0.7437800\ttotal: 39.4ms\tremaining: 618ms\n",
      "6:\tlearn: 0.6735459\ttotal: 46ms\tremaining: 612ms\n",
      "7:\tlearn: 0.6193337\ttotal: 51.8ms\tremaining: 595ms\n",
      "8:\tlearn: 0.5684535\ttotal: 58.3ms\tremaining: 590ms\n",
      "9:\tlearn: 0.5225109\ttotal: 64.7ms\tremaining: 582ms\n",
      "10:\tlearn: 0.4871916\ttotal: 71.6ms\tremaining: 579ms\n",
      "11:\tlearn: 0.4509628\ttotal: 77.9ms\tremaining: 571ms\n",
      "12:\tlearn: 0.4203315\ttotal: 85ms\tremaining: 569ms\n",
      "13:\tlearn: 0.3947165\ttotal: 91.9ms\tremaining: 564ms\n",
      "14:\tlearn: 0.3732858\ttotal: 98.7ms\tremaining: 559ms\n",
      "15:\tlearn: 0.3507724\ttotal: 105ms\tremaining: 550ms\n",
      "16:\tlearn: 0.3314522\ttotal: 111ms\tremaining: 540ms\n",
      "17:\tlearn: 0.3143572\ttotal: 117ms\tremaining: 534ms\n",
      "18:\tlearn: 0.2983180\ttotal: 124ms\tremaining: 529ms\n",
      "19:\tlearn: 0.2841852\ttotal: 130ms\tremaining: 520ms\n",
      "20:\tlearn: 0.2730686\ttotal: 136ms\tremaining: 510ms\n",
      "21:\tlearn: 0.2603782\ttotal: 141ms\tremaining: 500ms\n",
      "22:\tlearn: 0.2502897\ttotal: 148ms\tremaining: 496ms\n",
      "23:\tlearn: 0.2408606\ttotal: 155ms\tremaining: 489ms\n",
      "24:\tlearn: 0.2318609\ttotal: 161ms\tremaining: 482ms\n",
      "25:\tlearn: 0.2237580\ttotal: 167ms\tremaining: 476ms\n",
      "26:\tlearn: 0.2161467\ttotal: 174ms\tremaining: 470ms\n",
      "27:\tlearn: 0.2079498\ttotal: 181ms\tremaining: 465ms\n",
      "28:\tlearn: 0.2003236\ttotal: 188ms\tremaining: 460ms\n",
      "29:\tlearn: 0.1941279\ttotal: 195ms\tremaining: 455ms\n",
      "30:\tlearn: 0.1880293\ttotal: 202ms\tremaining: 451ms\n",
      "31:\tlearn: 0.1836837\ttotal: 209ms\tremaining: 444ms\n",
      "32:\tlearn: 0.1776616\ttotal: 215ms\tremaining: 436ms\n",
      "33:\tlearn: 0.1733883\ttotal: 221ms\tremaining: 428ms\n",
      "34:\tlearn: 0.1693205\ttotal: 226ms\tremaining: 420ms\n",
      "35:\tlearn: 0.1653322\ttotal: 232ms\tremaining: 413ms\n",
      "36:\tlearn: 0.1621920\ttotal: 238ms\tremaining: 405ms\n",
      "37:\tlearn: 0.1583416\ttotal: 243ms\tremaining: 396ms\n",
      "38:\tlearn: 0.1541215\ttotal: 248ms\tremaining: 388ms\n",
      "39:\tlearn: 0.1516859\ttotal: 253ms\tremaining: 379ms\n",
      "40:\tlearn: 0.1473107\ttotal: 258ms\tremaining: 371ms\n",
      "41:\tlearn: 0.1444466\ttotal: 263ms\tremaining: 363ms\n",
      "42:\tlearn: 0.1423218\ttotal: 268ms\tremaining: 355ms\n",
      "43:\tlearn: 0.1395213\ttotal: 273ms\tremaining: 347ms\n",
      "44:\tlearn: 0.1375772\ttotal: 277ms\tremaining: 339ms\n",
      "45:\tlearn: 0.1350098\ttotal: 283ms\tremaining: 332ms\n",
      "46:\tlearn: 0.1321990\ttotal: 288ms\tremaining: 325ms\n",
      "47:\tlearn: 0.1297481\ttotal: 292ms\tremaining: 317ms\n",
      "48:\tlearn: 0.1272453\ttotal: 298ms\tremaining: 310ms\n",
      "49:\tlearn: 0.1240709\ttotal: 303ms\tremaining: 303ms\n",
      "50:\tlearn: 0.1220976\ttotal: 308ms\tremaining: 295ms\n",
      "51:\tlearn: 0.1204160\ttotal: 312ms\tremaining: 288ms\n",
      "52:\tlearn: 0.1179875\ttotal: 317ms\tremaining: 281ms\n",
      "53:\tlearn: 0.1155918\ttotal: 322ms\tremaining: 274ms\n",
      "54:\tlearn: 0.1144902\ttotal: 327ms\tremaining: 267ms\n",
      "55:\tlearn: 0.1133022\ttotal: 332ms\tremaining: 261ms\n",
      "56:\tlearn: 0.1108260\ttotal: 337ms\tremaining: 254ms\n",
      "57:\tlearn: 0.1089310\ttotal: 342ms\tremaining: 247ms\n",
      "58:\tlearn: 0.1074156\ttotal: 346ms\tremaining: 241ms\n",
      "59:\tlearn: 0.1060477\ttotal: 352ms\tremaining: 234ms\n",
      "60:\tlearn: 0.1050489\ttotal: 356ms\tremaining: 228ms\n",
      "61:\tlearn: 0.1039062\ttotal: 361ms\tremaining: 221ms\n",
      "62:\tlearn: 0.1029145\ttotal: 366ms\tremaining: 215ms\n",
      "63:\tlearn: 0.1013139\ttotal: 371ms\tremaining: 209ms\n",
      "64:\tlearn: 0.1003145\ttotal: 376ms\tremaining: 202ms\n",
      "65:\tlearn: 0.0989576\ttotal: 380ms\tremaining: 196ms\n",
      "66:\tlearn: 0.0975547\ttotal: 385ms\tremaining: 190ms\n",
      "67:\tlearn: 0.0965074\ttotal: 390ms\tremaining: 183ms\n",
      "68:\tlearn: 0.0944328\ttotal: 394ms\tremaining: 177ms\n",
      "69:\tlearn: 0.0930039\ttotal: 399ms\tremaining: 171ms\n",
      "70:\tlearn: 0.0922131\ttotal: 404ms\tremaining: 165ms\n",
      "71:\tlearn: 0.0910582\ttotal: 408ms\tremaining: 159ms\n",
      "72:\tlearn: 0.0901636\ttotal: 415ms\tremaining: 154ms\n",
      "73:\tlearn: 0.0894056\ttotal: 421ms\tremaining: 148ms\n",
      "74:\tlearn: 0.0887467\ttotal: 427ms\tremaining: 142ms\n",
      "75:\tlearn: 0.0875115\ttotal: 432ms\tremaining: 137ms\n",
      "76:\tlearn: 0.0868935\ttotal: 439ms\tremaining: 131ms\n",
      "77:\tlearn: 0.0852968\ttotal: 445ms\tremaining: 125ms\n",
      "78:\tlearn: 0.0845763\ttotal: 451ms\tremaining: 120ms\n",
      "79:\tlearn: 0.0833380\ttotal: 457ms\tremaining: 114ms\n",
      "80:\tlearn: 0.0825764\ttotal: 463ms\tremaining: 109ms\n",
      "81:\tlearn: 0.0819221\ttotal: 470ms\tremaining: 103ms\n",
      "82:\tlearn: 0.0811593\ttotal: 475ms\tremaining: 97.3ms\n",
      "83:\tlearn: 0.0800566\ttotal: 480ms\tremaining: 91.5ms\n",
      "84:\tlearn: 0.0794021\ttotal: 486ms\tremaining: 85.8ms\n",
      "85:\tlearn: 0.0787560\ttotal: 492ms\tremaining: 80.1ms\n",
      "86:\tlearn: 0.0776676\ttotal: 498ms\tremaining: 74.5ms\n",
      "87:\tlearn: 0.0770668\ttotal: 505ms\tremaining: 68.9ms\n",
      "88:\tlearn: 0.0765102\ttotal: 511ms\tremaining: 63.1ms\n",
      "89:\tlearn: 0.0754987\ttotal: 516ms\tremaining: 57.4ms\n",
      "90:\tlearn: 0.0749628\ttotal: 522ms\tremaining: 51.6ms\n",
      "91:\tlearn: 0.0741834\ttotal: 527ms\tremaining: 45.8ms\n",
      "92:\tlearn: 0.0736212\ttotal: 533ms\tremaining: 40.1ms\n",
      "93:\tlearn: 0.0730004\ttotal: 539ms\tremaining: 34.4ms\n",
      "94:\tlearn: 0.0725432\ttotal: 545ms\tremaining: 28.7ms\n",
      "95:\tlearn: 0.0720607\ttotal: 552ms\tremaining: 23ms\n",
      "96:\tlearn: 0.0710572\ttotal: 558ms\tremaining: 17.3ms\n",
      "97:\tlearn: 0.0703171\ttotal: 565ms\tremaining: 11.5ms\n",
      "98:\tlearn: 0.0698560\ttotal: 572ms\tremaining: 5.78ms\n",
      "99:\tlearn: 0.0692224\ttotal: 579ms\tremaining: 0us\n",
      "0:\tlearn: 1.3388767\ttotal: 6.44ms\tremaining: 638ms\n",
      "1:\tlearn: 1.1487401\ttotal: 13.4ms\tremaining: 657ms\n",
      "2:\tlearn: 1.0023182\ttotal: 20.8ms\tremaining: 673ms\n",
      "3:\tlearn: 0.8882955\ttotal: 28ms\tremaining: 671ms\n",
      "4:\tlearn: 0.7992450\ttotal: 35.5ms\tremaining: 675ms\n",
      "5:\tlearn: 0.7322720\ttotal: 42ms\tremaining: 657ms\n",
      "6:\tlearn: 0.6698636\ttotal: 48.1ms\tremaining: 639ms\n",
      "7:\tlearn: 0.6172280\ttotal: 53.9ms\tremaining: 620ms\n",
      "8:\tlearn: 0.5696343\ttotal: 59.2ms\tremaining: 599ms\n",
      "9:\tlearn: 0.5242698\ttotal: 64.4ms\tremaining: 580ms\n",
      "10:\tlearn: 0.4889411\ttotal: 70.5ms\tremaining: 570ms\n",
      "11:\tlearn: 0.4541010\ttotal: 76ms\tremaining: 558ms\n",
      "12:\tlearn: 0.4223826\ttotal: 81.8ms\tremaining: 548ms\n",
      "13:\tlearn: 0.3945454\ttotal: 87.3ms\tremaining: 536ms\n",
      "14:\tlearn: 0.3720522\ttotal: 92.6ms\tremaining: 525ms\n",
      "15:\tlearn: 0.3503785\ttotal: 97.8ms\tremaining: 514ms\n",
      "16:\tlearn: 0.3310588\ttotal: 104ms\tremaining: 505ms\n",
      "17:\tlearn: 0.3120114\ttotal: 109ms\tremaining: 497ms\n",
      "18:\tlearn: 0.2969690\ttotal: 114ms\tremaining: 487ms\n",
      "19:\tlearn: 0.2849573\ttotal: 120ms\tremaining: 480ms\n",
      "20:\tlearn: 0.2736807\ttotal: 126ms\tremaining: 473ms\n",
      "21:\tlearn: 0.2618974\ttotal: 131ms\tremaining: 464ms\n",
      "22:\tlearn: 0.2509617\ttotal: 136ms\tremaining: 457ms\n",
      "23:\tlearn: 0.2398802\ttotal: 142ms\tremaining: 450ms\n",
      "24:\tlearn: 0.2304147\ttotal: 147ms\tremaining: 441ms\n",
      "25:\tlearn: 0.2217033\ttotal: 153ms\tremaining: 435ms\n",
      "26:\tlearn: 0.2124680\ttotal: 158ms\tremaining: 427ms\n",
      "27:\tlearn: 0.2059719\ttotal: 163ms\tremaining: 418ms\n",
      "28:\tlearn: 0.1987241\ttotal: 168ms\tremaining: 410ms\n",
      "29:\tlearn: 0.1913302\ttotal: 173ms\tremaining: 404ms\n",
      "30:\tlearn: 0.1860386\ttotal: 178ms\tremaining: 397ms\n",
      "31:\tlearn: 0.1814542\ttotal: 184ms\tremaining: 390ms\n",
      "32:\tlearn: 0.1750098\ttotal: 189ms\tremaining: 383ms\n",
      "33:\tlearn: 0.1697118\ttotal: 193ms\tremaining: 375ms\n",
      "34:\tlearn: 0.1650141\ttotal: 198ms\tremaining: 368ms\n",
      "35:\tlearn: 0.1602958\ttotal: 203ms\tremaining: 360ms\n",
      "36:\tlearn: 0.1568501\ttotal: 208ms\tremaining: 353ms\n",
      "37:\tlearn: 0.1526871\ttotal: 212ms\tremaining: 347ms\n",
      "38:\tlearn: 0.1481213\ttotal: 218ms\tremaining: 340ms\n",
      "39:\tlearn: 0.1444830\ttotal: 222ms\tremaining: 333ms\n",
      "40:\tlearn: 0.1415577\ttotal: 228ms\tremaining: 328ms\n",
      "41:\tlearn: 0.1389945\ttotal: 233ms\tremaining: 321ms\n",
      "42:\tlearn: 0.1357089\ttotal: 238ms\tremaining: 315ms\n",
      "43:\tlearn: 0.1328721\ttotal: 242ms\tremaining: 308ms\n",
      "44:\tlearn: 0.1295832\ttotal: 247ms\tremaining: 302ms\n",
      "45:\tlearn: 0.1270247\ttotal: 252ms\tremaining: 296ms\n",
      "46:\tlearn: 0.1248728\ttotal: 257ms\tremaining: 290ms\n",
      "47:\tlearn: 0.1226723\ttotal: 262ms\tremaining: 284ms\n",
      "48:\tlearn: 0.1210259\ttotal: 267ms\tremaining: 278ms\n",
      "49:\tlearn: 0.1188957\ttotal: 272ms\tremaining: 272ms\n",
      "50:\tlearn: 0.1171153\ttotal: 277ms\tremaining: 266ms\n",
      "51:\tlearn: 0.1155149\ttotal: 282ms\tremaining: 260ms\n",
      "52:\tlearn: 0.1129947\ttotal: 287ms\tremaining: 254ms\n",
      "53:\tlearn: 0.1105470\ttotal: 292ms\tremaining: 249ms\n",
      "54:\tlearn: 0.1087533\ttotal: 297ms\tremaining: 243ms\n",
      "55:\tlearn: 0.1074453\ttotal: 302ms\tremaining: 237ms\n",
      "56:\tlearn: 0.1057290\ttotal: 307ms\tremaining: 231ms\n",
      "57:\tlearn: 0.1038413\ttotal: 311ms\tremaining: 226ms\n",
      "58:\tlearn: 0.1023716\ttotal: 317ms\tremaining: 220ms\n",
      "59:\tlearn: 0.1010621\ttotal: 321ms\tremaining: 214ms\n",
      "60:\tlearn: 0.0994461\ttotal: 326ms\tremaining: 208ms\n",
      "61:\tlearn: 0.0982647\ttotal: 331ms\tremaining: 203ms\n",
      "62:\tlearn: 0.0974863\ttotal: 335ms\tremaining: 197ms\n",
      "63:\tlearn: 0.0964839\ttotal: 340ms\tremaining: 191ms\n",
      "64:\tlearn: 0.0953782\ttotal: 344ms\tremaining: 185ms\n",
      "65:\tlearn: 0.0945476\ttotal: 350ms\tremaining: 180ms\n",
      "66:\tlearn: 0.0934406\ttotal: 355ms\tremaining: 175ms\n",
      "67:\tlearn: 0.0924483\ttotal: 360ms\tremaining: 169ms\n",
      "68:\tlearn: 0.0913619\ttotal: 365ms\tremaining: 164ms\n",
      "69:\tlearn: 0.0900247\ttotal: 370ms\tremaining: 159ms\n",
      "70:\tlearn: 0.0891525\ttotal: 375ms\tremaining: 153ms\n",
      "71:\tlearn: 0.0879704\ttotal: 381ms\tremaining: 148ms\n",
      "72:\tlearn: 0.0871147\ttotal: 386ms\tremaining: 143ms\n",
      "73:\tlearn: 0.0864231\ttotal: 391ms\tremaining: 137ms\n",
      "74:\tlearn: 0.0856759\ttotal: 396ms\tremaining: 132ms\n",
      "75:\tlearn: 0.0849133\ttotal: 402ms\tremaining: 127ms\n",
      "76:\tlearn: 0.0837909\ttotal: 407ms\tremaining: 122ms\n",
      "77:\tlearn: 0.0829574\ttotal: 412ms\tremaining: 116ms\n",
      "78:\tlearn: 0.0820911\ttotal: 417ms\tremaining: 111ms\n",
      "79:\tlearn: 0.0816497\ttotal: 422ms\tremaining: 106ms\n",
      "80:\tlearn: 0.0805394\ttotal: 427ms\tremaining: 100ms\n",
      "81:\tlearn: 0.0800306\ttotal: 432ms\tremaining: 94.9ms\n",
      "82:\tlearn: 0.0792604\ttotal: 437ms\tremaining: 89.5ms\n",
      "83:\tlearn: 0.0785444\ttotal: 443ms\tremaining: 84.4ms\n",
      "84:\tlearn: 0.0778941\ttotal: 448ms\tremaining: 79.1ms\n",
      "85:\tlearn: 0.0769019\ttotal: 455ms\tremaining: 74ms\n",
      "86:\tlearn: 0.0763480\ttotal: 461ms\tremaining: 68.9ms\n",
      "87:\tlearn: 0.0754984\ttotal: 466ms\tremaining: 63.6ms\n",
      "88:\tlearn: 0.0748299\ttotal: 472ms\tremaining: 58.3ms\n",
      "89:\tlearn: 0.0736158\ttotal: 477ms\tremaining: 53ms\n",
      "90:\tlearn: 0.0729759\ttotal: 482ms\tremaining: 47.7ms\n",
      "91:\tlearn: 0.0723379\ttotal: 488ms\tremaining: 42.5ms\n",
      "92:\tlearn: 0.0717777\ttotal: 494ms\tremaining: 37.2ms\n",
      "93:\tlearn: 0.0714221\ttotal: 500ms\tremaining: 31.9ms\n",
      "94:\tlearn: 0.0705927\ttotal: 506ms\tremaining: 26.6ms\n",
      "95:\tlearn: 0.0700686\ttotal: 511ms\tremaining: 21.3ms\n",
      "96:\tlearn: 0.0689482\ttotal: 516ms\tremaining: 16ms\n",
      "97:\tlearn: 0.0685753\ttotal: 522ms\tremaining: 10.7ms\n",
      "98:\tlearn: 0.0677069\ttotal: 527ms\tremaining: 5.32ms\n",
      "99:\tlearn: 0.0671704\ttotal: 532ms\tremaining: 0us\n",
      "0:\tlearn: 1.3384999\ttotal: 7.31ms\tremaining: 724ms\n",
      "1:\tlearn: 1.1532619\ttotal: 13.8ms\tremaining: 674ms\n",
      "2:\tlearn: 1.0122540\ttotal: 20.3ms\tremaining: 655ms\n",
      "3:\tlearn: 0.8975687\ttotal: 27.2ms\tremaining: 653ms\n",
      "4:\tlearn: 0.8124224\ttotal: 33.6ms\tremaining: 639ms\n",
      "5:\tlearn: 0.7424668\ttotal: 39ms\tremaining: 611ms\n",
      "6:\tlearn: 0.6789661\ttotal: 46.1ms\tremaining: 613ms\n",
      "7:\tlearn: 0.6248663\ttotal: 53.9ms\tremaining: 620ms\n",
      "8:\tlearn: 0.5773513\ttotal: 60.1ms\tremaining: 608ms\n",
      "9:\tlearn: 0.5345901\ttotal: 67.3ms\tremaining: 606ms\n",
      "10:\tlearn: 0.4945393\ttotal: 73.6ms\tremaining: 596ms\n",
      "11:\tlearn: 0.4623025\ttotal: 80.1ms\tremaining: 587ms\n",
      "12:\tlearn: 0.4323802\ttotal: 86.5ms\tremaining: 579ms\n",
      "13:\tlearn: 0.4067834\ttotal: 93.5ms\tremaining: 574ms\n",
      "14:\tlearn: 0.3835607\ttotal: 99.8ms\tremaining: 566ms\n",
      "15:\tlearn: 0.3614974\ttotal: 106ms\tremaining: 555ms\n",
      "16:\tlearn: 0.3400022\ttotal: 111ms\tremaining: 544ms\n",
      "17:\tlearn: 0.3206307\ttotal: 118ms\tremaining: 536ms\n",
      "18:\tlearn: 0.3042533\ttotal: 124ms\tremaining: 530ms\n",
      "19:\tlearn: 0.2896401\ttotal: 132ms\tremaining: 528ms\n",
      "20:\tlearn: 0.2786637\ttotal: 138ms\tremaining: 519ms\n",
      "21:\tlearn: 0.2662831\ttotal: 144ms\tremaining: 511ms\n",
      "22:\tlearn: 0.2547776\ttotal: 149ms\tremaining: 500ms\n",
      "23:\tlearn: 0.2437282\ttotal: 156ms\tremaining: 492ms\n",
      "24:\tlearn: 0.2335298\ttotal: 163ms\tremaining: 488ms\n",
      "25:\tlearn: 0.2262043\ttotal: 169ms\tremaining: 482ms\n",
      "26:\tlearn: 0.2166388\ttotal: 176ms\tremaining: 476ms\n",
      "27:\tlearn: 0.2076452\ttotal: 182ms\tremaining: 469ms\n",
      "28:\tlearn: 0.2025111\ttotal: 188ms\tremaining: 461ms\n",
      "29:\tlearn: 0.1951975\ttotal: 194ms\tremaining: 453ms\n",
      "30:\tlearn: 0.1896546\ttotal: 201ms\tremaining: 448ms\n",
      "31:\tlearn: 0.1848473\ttotal: 208ms\tremaining: 442ms\n",
      "32:\tlearn: 0.1795029\ttotal: 215ms\tremaining: 436ms\n",
      "33:\tlearn: 0.1755513\ttotal: 222ms\tremaining: 430ms\n",
      "34:\tlearn: 0.1700373\ttotal: 228ms\tremaining: 423ms\n",
      "35:\tlearn: 0.1662271\ttotal: 234ms\tremaining: 416ms\n",
      "36:\tlearn: 0.1625469\ttotal: 239ms\tremaining: 407ms\n",
      "37:\tlearn: 0.1584040\ttotal: 244ms\tremaining: 398ms\n",
      "38:\tlearn: 0.1558628\ttotal: 249ms\tremaining: 390ms\n",
      "39:\tlearn: 0.1518201\ttotal: 257ms\tremaining: 385ms\n",
      "40:\tlearn: 0.1475112\ttotal: 262ms\tremaining: 378ms\n",
      "41:\tlearn: 0.1440863\ttotal: 268ms\tremaining: 370ms\n",
      "42:\tlearn: 0.1409193\ttotal: 274ms\tremaining: 363ms\n",
      "43:\tlearn: 0.1375779\ttotal: 279ms\tremaining: 355ms\n",
      "44:\tlearn: 0.1339378\ttotal: 284ms\tremaining: 348ms\n",
      "45:\tlearn: 0.1312502\ttotal: 291ms\tremaining: 341ms\n",
      "46:\tlearn: 0.1286866\ttotal: 296ms\tremaining: 334ms\n",
      "47:\tlearn: 0.1266638\ttotal: 303ms\tremaining: 328ms\n",
      "48:\tlearn: 0.1251068\ttotal: 310ms\tremaining: 322ms\n",
      "49:\tlearn: 0.1226799\ttotal: 315ms\tremaining: 315ms\n",
      "50:\tlearn: 0.1203469\ttotal: 321ms\tremaining: 308ms\n",
      "51:\tlearn: 0.1185872\ttotal: 326ms\tremaining: 301ms\n",
      "52:\tlearn: 0.1164041\ttotal: 332ms\tremaining: 294ms\n",
      "53:\tlearn: 0.1144278\ttotal: 337ms\tremaining: 287ms\n",
      "54:\tlearn: 0.1128813\ttotal: 342ms\tremaining: 280ms\n",
      "55:\tlearn: 0.1117688\ttotal: 348ms\tremaining: 274ms\n",
      "56:\tlearn: 0.1095720\ttotal: 354ms\tremaining: 267ms\n",
      "57:\tlearn: 0.1077926\ttotal: 360ms\tremaining: 261ms\n",
      "58:\tlearn: 0.1067400\ttotal: 365ms\tremaining: 254ms\n",
      "59:\tlearn: 0.1055537\ttotal: 371ms\tremaining: 248ms\n",
      "60:\tlearn: 0.1041786\ttotal: 377ms\tremaining: 241ms\n",
      "61:\tlearn: 0.1029065\ttotal: 383ms\tremaining: 235ms\n",
      "62:\tlearn: 0.1021913\ttotal: 388ms\tremaining: 228ms\n",
      "63:\tlearn: 0.1007608\ttotal: 394ms\tremaining: 221ms\n",
      "64:\tlearn: 0.0997579\ttotal: 399ms\tremaining: 215ms\n",
      "65:\tlearn: 0.0990673\ttotal: 405ms\tremaining: 209ms\n",
      "66:\tlearn: 0.0968445\ttotal: 410ms\tremaining: 202ms\n",
      "67:\tlearn: 0.0952813\ttotal: 416ms\tremaining: 196ms\n",
      "68:\tlearn: 0.0937688\ttotal: 422ms\tremaining: 189ms\n",
      "69:\tlearn: 0.0924327\ttotal: 428ms\tremaining: 183ms\n",
      "70:\tlearn: 0.0913693\ttotal: 434ms\tremaining: 177ms\n",
      "71:\tlearn: 0.0904833\ttotal: 440ms\tremaining: 171ms\n",
      "72:\tlearn: 0.0896981\ttotal: 446ms\tremaining: 165ms\n",
      "73:\tlearn: 0.0890653\ttotal: 452ms\tremaining: 159ms\n",
      "74:\tlearn: 0.0880121\ttotal: 457ms\tremaining: 152ms\n",
      "75:\tlearn: 0.0867584\ttotal: 464ms\tremaining: 146ms\n",
      "76:\tlearn: 0.0859553\ttotal: 470ms\tremaining: 140ms\n",
      "77:\tlearn: 0.0847172\ttotal: 476ms\tremaining: 134ms\n",
      "78:\tlearn: 0.0842354\ttotal: 483ms\tremaining: 128ms\n",
      "79:\tlearn: 0.0831690\ttotal: 489ms\tremaining: 122ms\n",
      "80:\tlearn: 0.0824627\ttotal: 495ms\tremaining: 116ms\n",
      "81:\tlearn: 0.0815714\ttotal: 502ms\tremaining: 110ms\n",
      "82:\tlearn: 0.0806681\ttotal: 509ms\tremaining: 104ms\n",
      "83:\tlearn: 0.0793243\ttotal: 515ms\tremaining: 98.2ms\n",
      "84:\tlearn: 0.0788763\ttotal: 522ms\tremaining: 92.1ms\n",
      "85:\tlearn: 0.0778944\ttotal: 528ms\tremaining: 86ms\n",
      "86:\tlearn: 0.0774153\ttotal: 534ms\tremaining: 79.7ms\n",
      "87:\tlearn: 0.0761912\ttotal: 540ms\tremaining: 73.6ms\n",
      "88:\tlearn: 0.0752979\ttotal: 545ms\tremaining: 67.4ms\n",
      "89:\tlearn: 0.0747008\ttotal: 552ms\tremaining: 61.4ms\n",
      "90:\tlearn: 0.0738462\ttotal: 559ms\tremaining: 55.3ms\n",
      "91:\tlearn: 0.0727823\ttotal: 566ms\tremaining: 49.2ms\n",
      "92:\tlearn: 0.0721488\ttotal: 572ms\tremaining: 43.1ms\n",
      "93:\tlearn: 0.0715942\ttotal: 578ms\tremaining: 36.9ms\n",
      "94:\tlearn: 0.0711114\ttotal: 584ms\tremaining: 30.7ms\n",
      "95:\tlearn: 0.0708339\ttotal: 590ms\tremaining: 24.6ms\n",
      "96:\tlearn: 0.0701920\ttotal: 595ms\tremaining: 18.4ms\n",
      "97:\tlearn: 0.0698052\ttotal: 600ms\tremaining: 12.2ms\n",
      "98:\tlearn: 0.0692408\ttotal: 606ms\tremaining: 6.12ms\n",
      "99:\tlearn: 0.0688345\ttotal: 611ms\tremaining: 0us\n",
      "0:\tlearn: 1.3285518\ttotal: 7.87ms\tremaining: 779ms\n",
      "1:\tlearn: 1.1423234\ttotal: 14ms\tremaining: 686ms\n",
      "2:\tlearn: 0.9968486\ttotal: 21.4ms\tremaining: 693ms\n",
      "3:\tlearn: 0.8779670\ttotal: 28.2ms\tremaining: 676ms\n",
      "4:\tlearn: 0.7890283\ttotal: 34.7ms\tremaining: 660ms\n",
      "5:\tlearn: 0.7183573\ttotal: 40.6ms\tremaining: 636ms\n",
      "6:\tlearn: 0.6549995\ttotal: 46.7ms\tremaining: 620ms\n",
      "7:\tlearn: 0.6004272\ttotal: 52.4ms\tremaining: 603ms\n",
      "8:\tlearn: 0.5548612\ttotal: 59ms\tremaining: 597ms\n",
      "9:\tlearn: 0.5124634\ttotal: 65.8ms\tremaining: 592ms\n",
      "10:\tlearn: 0.4755672\ttotal: 71.8ms\tremaining: 581ms\n",
      "11:\tlearn: 0.4403042\ttotal: 78ms\tremaining: 572ms\n",
      "12:\tlearn: 0.4116800\ttotal: 83.6ms\tremaining: 559ms\n",
      "13:\tlearn: 0.3839759\ttotal: 89.5ms\tremaining: 550ms\n",
      "14:\tlearn: 0.3618755\ttotal: 97.2ms\tremaining: 551ms\n",
      "15:\tlearn: 0.3407021\ttotal: 103ms\tremaining: 543ms\n",
      "16:\tlearn: 0.3195274\ttotal: 110ms\tremaining: 539ms\n",
      "17:\tlearn: 0.3000218\ttotal: 117ms\tremaining: 532ms\n",
      "18:\tlearn: 0.2854729\ttotal: 123ms\tremaining: 524ms\n",
      "19:\tlearn: 0.2730079\ttotal: 129ms\tremaining: 515ms\n",
      "20:\tlearn: 0.2622091\ttotal: 134ms\tremaining: 505ms\n",
      "21:\tlearn: 0.2490883\ttotal: 140ms\tremaining: 497ms\n",
      "22:\tlearn: 0.2389464\ttotal: 145ms\tremaining: 486ms\n",
      "23:\tlearn: 0.2288747\ttotal: 150ms\tremaining: 476ms\n",
      "24:\tlearn: 0.2187963\ttotal: 155ms\tremaining: 466ms\n",
      "25:\tlearn: 0.2105353\ttotal: 161ms\tremaining: 457ms\n",
      "26:\tlearn: 0.2005839\ttotal: 166ms\tremaining: 448ms\n",
      "27:\tlearn: 0.1921377\ttotal: 170ms\tremaining: 438ms\n",
      "28:\tlearn: 0.1855623\ttotal: 176ms\tremaining: 432ms\n",
      "29:\tlearn: 0.1795007\ttotal: 181ms\tremaining: 423ms\n",
      "30:\tlearn: 0.1735893\ttotal: 187ms\tremaining: 416ms\n",
      "31:\tlearn: 0.1681252\ttotal: 191ms\tremaining: 407ms\n",
      "32:\tlearn: 0.1627741\ttotal: 196ms\tremaining: 398ms\n",
      "33:\tlearn: 0.1588402\ttotal: 202ms\tremaining: 392ms\n",
      "34:\tlearn: 0.1546885\ttotal: 207ms\tremaining: 385ms\n",
      "35:\tlearn: 0.1505070\ttotal: 212ms\tremaining: 378ms\n",
      "36:\tlearn: 0.1474326\ttotal: 217ms\tremaining: 370ms\n",
      "37:\tlearn: 0.1436143\ttotal: 224ms\tremaining: 365ms\n",
      "38:\tlearn: 0.1396540\ttotal: 230ms\tremaining: 359ms\n",
      "39:\tlearn: 0.1365725\ttotal: 235ms\tremaining: 352ms\n",
      "40:\tlearn: 0.1331760\ttotal: 240ms\tremaining: 345ms\n",
      "41:\tlearn: 0.1306183\ttotal: 245ms\tremaining: 338ms\n",
      "42:\tlearn: 0.1278260\ttotal: 251ms\tremaining: 333ms\n",
      "43:\tlearn: 0.1263661\ttotal: 256ms\tremaining: 326ms\n",
      "44:\tlearn: 0.1238032\ttotal: 261ms\tremaining: 319ms\n",
      "45:\tlearn: 0.1218783\ttotal: 267ms\tremaining: 314ms\n",
      "46:\tlearn: 0.1189392\ttotal: 272ms\tremaining: 307ms\n",
      "47:\tlearn: 0.1175823\ttotal: 278ms\tremaining: 301ms\n",
      "48:\tlearn: 0.1159053\ttotal: 283ms\tremaining: 295ms\n",
      "49:\tlearn: 0.1137809\ttotal: 289ms\tremaining: 289ms\n",
      "50:\tlearn: 0.1122875\ttotal: 294ms\tremaining: 283ms\n",
      "51:\tlearn: 0.1110358\ttotal: 300ms\tremaining: 277ms\n",
      "52:\tlearn: 0.1089368\ttotal: 306ms\tremaining: 271ms\n",
      "53:\tlearn: 0.1078784\ttotal: 311ms\tremaining: 265ms\n",
      "54:\tlearn: 0.1063643\ttotal: 316ms\tremaining: 259ms\n",
      "55:\tlearn: 0.1056205\ttotal: 321ms\tremaining: 252ms\n",
      "56:\tlearn: 0.1032734\ttotal: 327ms\tremaining: 247ms\n",
      "57:\tlearn: 0.1020389\ttotal: 333ms\tremaining: 241ms\n",
      "58:\tlearn: 0.0998150\ttotal: 338ms\tremaining: 235ms\n",
      "59:\tlearn: 0.0982552\ttotal: 343ms\tremaining: 229ms\n",
      "60:\tlearn: 0.0971866\ttotal: 348ms\tremaining: 223ms\n",
      "61:\tlearn: 0.0961067\ttotal: 353ms\tremaining: 216ms\n",
      "62:\tlearn: 0.0951756\ttotal: 358ms\tremaining: 210ms\n",
      "63:\tlearn: 0.0940757\ttotal: 363ms\tremaining: 204ms\n",
      "64:\tlearn: 0.0928880\ttotal: 368ms\tremaining: 198ms\n",
      "65:\tlearn: 0.0920117\ttotal: 373ms\tremaining: 192ms\n",
      "66:\tlearn: 0.0911510\ttotal: 378ms\tremaining: 186ms\n",
      "67:\tlearn: 0.0898216\ttotal: 383ms\tremaining: 180ms\n",
      "68:\tlearn: 0.0888518\ttotal: 387ms\tremaining: 174ms\n",
      "69:\tlearn: 0.0878853\ttotal: 392ms\tremaining: 168ms\n",
      "70:\tlearn: 0.0870877\ttotal: 397ms\tremaining: 162ms\n",
      "71:\tlearn: 0.0858608\ttotal: 401ms\tremaining: 156ms\n",
      "72:\tlearn: 0.0851576\ttotal: 407ms\tremaining: 150ms\n",
      "73:\tlearn: 0.0844604\ttotal: 412ms\tremaining: 145ms\n",
      "74:\tlearn: 0.0834442\ttotal: 417ms\tremaining: 139ms\n",
      "75:\tlearn: 0.0824678\ttotal: 422ms\tremaining: 133ms\n",
      "76:\tlearn: 0.0816738\ttotal: 427ms\tremaining: 128ms\n",
      "77:\tlearn: 0.0810822\ttotal: 432ms\tremaining: 122ms\n",
      "78:\tlearn: 0.0798055\ttotal: 437ms\tremaining: 116ms\n",
      "79:\tlearn: 0.0791260\ttotal: 443ms\tremaining: 111ms\n",
      "80:\tlearn: 0.0786768\ttotal: 448ms\tremaining: 105ms\n",
      "81:\tlearn: 0.0783418\ttotal: 451ms\tremaining: 99ms\n",
      "82:\tlearn: 0.0780098\ttotal: 456ms\tremaining: 93.5ms\n",
      "83:\tlearn: 0.0776353\ttotal: 461ms\tremaining: 87.8ms\n",
      "84:\tlearn: 0.0773013\ttotal: 467ms\tremaining: 82.3ms\n",
      "85:\tlearn: 0.0764590\ttotal: 472ms\tremaining: 76.8ms\n",
      "86:\tlearn: 0.0758740\ttotal: 477ms\tremaining: 71.3ms\n",
      "87:\tlearn: 0.0753950\ttotal: 482ms\tremaining: 65.7ms\n",
      "88:\tlearn: 0.0748859\ttotal: 486ms\tremaining: 60.1ms\n",
      "89:\tlearn: 0.0740584\ttotal: 491ms\tremaining: 54.6ms\n",
      "90:\tlearn: 0.0735214\ttotal: 496ms\tremaining: 49ms\n",
      "91:\tlearn: 0.0731156\ttotal: 500ms\tremaining: 43.5ms\n",
      "92:\tlearn: 0.0726189\ttotal: 505ms\tremaining: 38ms\n",
      "93:\tlearn: 0.0717604\ttotal: 511ms\tremaining: 32.6ms\n",
      "94:\tlearn: 0.0713078\ttotal: 515ms\tremaining: 27.1ms\n",
      "95:\tlearn: 0.0701741\ttotal: 521ms\tremaining: 21.7ms\n",
      "96:\tlearn: 0.0697695\ttotal: 526ms\tremaining: 16.3ms\n",
      "97:\tlearn: 0.0693079\ttotal: 530ms\tremaining: 10.8ms\n",
      "98:\tlearn: 0.0688670\ttotal: 535ms\tremaining: 5.41ms\n",
      "99:\tlearn: 0.0683547\ttotal: 541ms\tremaining: 0us\n",
      "0:\tlearn: 1.3306975\ttotal: 7ms\tremaining: 693ms\n",
      "1:\tlearn: 1.1403004\ttotal: 14.4ms\tremaining: 705ms\n",
      "2:\tlearn: 1.0005725\ttotal: 21.3ms\tremaining: 690ms\n",
      "3:\tlearn: 0.8914900\ttotal: 27.6ms\tremaining: 663ms\n",
      "4:\tlearn: 0.8078629\ttotal: 34.3ms\tremaining: 653ms\n",
      "5:\tlearn: 0.7386735\ttotal: 40.6ms\tremaining: 637ms\n",
      "6:\tlearn: 0.6782959\ttotal: 46.5ms\tremaining: 617ms\n",
      "7:\tlearn: 0.6199823\ttotal: 52.9ms\tremaining: 608ms\n",
      "8:\tlearn: 0.5725618\ttotal: 58.9ms\tremaining: 596ms\n",
      "9:\tlearn: 0.5314758\ttotal: 65.2ms\tremaining: 587ms\n",
      "10:\tlearn: 0.4903178\ttotal: 71ms\tremaining: 575ms\n",
      "11:\tlearn: 0.4540166\ttotal: 76.7ms\tremaining: 562ms\n",
      "12:\tlearn: 0.4225885\ttotal: 81.7ms\tremaining: 547ms\n",
      "13:\tlearn: 0.3967610\ttotal: 86.6ms\tremaining: 532ms\n",
      "14:\tlearn: 0.3728201\ttotal: 92.2ms\tremaining: 523ms\n",
      "15:\tlearn: 0.3507383\ttotal: 98ms\tremaining: 515ms\n",
      "16:\tlearn: 0.3288318\ttotal: 103ms\tremaining: 502ms\n",
      "17:\tlearn: 0.3111860\ttotal: 109ms\tremaining: 495ms\n",
      "18:\tlearn: 0.2959350\ttotal: 114ms\tremaining: 487ms\n",
      "19:\tlearn: 0.2816671\ttotal: 121ms\tremaining: 482ms\n",
      "20:\tlearn: 0.2688536\ttotal: 127ms\tremaining: 477ms\n",
      "21:\tlearn: 0.2553908\ttotal: 133ms\tremaining: 470ms\n",
      "22:\tlearn: 0.2437088\ttotal: 139ms\tremaining: 465ms\n",
      "23:\tlearn: 0.2346525\ttotal: 145ms\tremaining: 459ms\n",
      "24:\tlearn: 0.2249474\ttotal: 152ms\tremaining: 455ms\n",
      "25:\tlearn: 0.2169519\ttotal: 157ms\tremaining: 448ms\n",
      "26:\tlearn: 0.2068644\ttotal: 163ms\tremaining: 441ms\n",
      "27:\tlearn: 0.1994248\ttotal: 169ms\tremaining: 435ms\n",
      "28:\tlearn: 0.1926102\ttotal: 175ms\tremaining: 429ms\n",
      "29:\tlearn: 0.1846950\ttotal: 183ms\tremaining: 426ms\n",
      "30:\tlearn: 0.1787331\ttotal: 189ms\tremaining: 420ms\n",
      "31:\tlearn: 0.1736039\ttotal: 195ms\tremaining: 415ms\n",
      "32:\tlearn: 0.1677597\ttotal: 201ms\tremaining: 408ms\n",
      "33:\tlearn: 0.1637803\ttotal: 206ms\tremaining: 401ms\n",
      "34:\tlearn: 0.1588409\ttotal: 213ms\tremaining: 395ms\n",
      "35:\tlearn: 0.1551210\ttotal: 220ms\tremaining: 391ms\n",
      "36:\tlearn: 0.1513296\ttotal: 226ms\tremaining: 385ms\n",
      "37:\tlearn: 0.1481444\ttotal: 233ms\tremaining: 379ms\n",
      "38:\tlearn: 0.1444100\ttotal: 239ms\tremaining: 373ms\n",
      "39:\tlearn: 0.1416670\ttotal: 245ms\tremaining: 367ms\n",
      "40:\tlearn: 0.1379300\ttotal: 251ms\tremaining: 362ms\n",
      "41:\tlearn: 0.1340995\ttotal: 257ms\tremaining: 354ms\n",
      "42:\tlearn: 0.1304399\ttotal: 262ms\tremaining: 347ms\n",
      "43:\tlearn: 0.1283736\ttotal: 267ms\tremaining: 340ms\n",
      "44:\tlearn: 0.1258773\ttotal: 273ms\tremaining: 333ms\n",
      "45:\tlearn: 0.1228675\ttotal: 278ms\tremaining: 326ms\n",
      "46:\tlearn: 0.1210978\ttotal: 282ms\tremaining: 318ms\n",
      "47:\tlearn: 0.1192733\ttotal: 288ms\tremaining: 311ms\n",
      "48:\tlearn: 0.1179102\ttotal: 292ms\tremaining: 304ms\n",
      "49:\tlearn: 0.1158861\ttotal: 298ms\tremaining: 298ms\n",
      "50:\tlearn: 0.1136296\ttotal: 304ms\tremaining: 292ms\n",
      "51:\tlearn: 0.1112634\ttotal: 309ms\tremaining: 285ms\n",
      "52:\tlearn: 0.1091150\ttotal: 314ms\tremaining: 278ms\n",
      "53:\tlearn: 0.1071438\ttotal: 320ms\tremaining: 273ms\n",
      "54:\tlearn: 0.1049627\ttotal: 326ms\tremaining: 267ms\n",
      "55:\tlearn: 0.1035677\ttotal: 331ms\tremaining: 260ms\n",
      "56:\tlearn: 0.1011580\ttotal: 336ms\tremaining: 253ms\n",
      "57:\tlearn: 0.0998047\ttotal: 341ms\tremaining: 247ms\n",
      "58:\tlearn: 0.0987154\ttotal: 346ms\tremaining: 240ms\n",
      "59:\tlearn: 0.0977354\ttotal: 351ms\tremaining: 234ms\n",
      "60:\tlearn: 0.0960081\ttotal: 356ms\tremaining: 228ms\n",
      "61:\tlearn: 0.0951948\ttotal: 361ms\tremaining: 221ms\n",
      "62:\tlearn: 0.0943178\ttotal: 366ms\tremaining: 215ms\n",
      "63:\tlearn: 0.0924301\ttotal: 371ms\tremaining: 209ms\n",
      "64:\tlearn: 0.0912664\ttotal: 377ms\tremaining: 203ms\n",
      "65:\tlearn: 0.0899462\ttotal: 383ms\tremaining: 197ms\n",
      "66:\tlearn: 0.0884065\ttotal: 389ms\tremaining: 191ms\n",
      "67:\tlearn: 0.0873245\ttotal: 394ms\tremaining: 185ms\n",
      "68:\tlearn: 0.0862308\ttotal: 399ms\tremaining: 179ms\n",
      "69:\tlearn: 0.0851794\ttotal: 405ms\tremaining: 174ms\n",
      "70:\tlearn: 0.0843863\ttotal: 411ms\tremaining: 168ms\n",
      "71:\tlearn: 0.0834109\ttotal: 417ms\tremaining: 162ms\n",
      "72:\tlearn: 0.0824132\ttotal: 422ms\tremaining: 156ms\n",
      "73:\tlearn: 0.0818229\ttotal: 428ms\tremaining: 150ms\n",
      "74:\tlearn: 0.0810589\ttotal: 435ms\tremaining: 145ms\n",
      "75:\tlearn: 0.0801421\ttotal: 441ms\tremaining: 139ms\n",
      "76:\tlearn: 0.0795095\ttotal: 448ms\tremaining: 134ms\n",
      "77:\tlearn: 0.0781740\ttotal: 454ms\tremaining: 128ms\n",
      "78:\tlearn: 0.0776617\ttotal: 460ms\tremaining: 122ms\n",
      "79:\tlearn: 0.0765272\ttotal: 466ms\tremaining: 117ms\n",
      "80:\tlearn: 0.0760241\ttotal: 472ms\tremaining: 111ms\n",
      "81:\tlearn: 0.0754676\ttotal: 478ms\tremaining: 105ms\n",
      "82:\tlearn: 0.0749162\ttotal: 483ms\tremaining: 99ms\n",
      "83:\tlearn: 0.0744221\ttotal: 489ms\tremaining: 93.2ms\n",
      "84:\tlearn: 0.0735985\ttotal: 495ms\tremaining: 87.4ms\n",
      "85:\tlearn: 0.0730556\ttotal: 501ms\tremaining: 81.5ms\n",
      "86:\tlearn: 0.0719953\ttotal: 507ms\tremaining: 75.8ms\n",
      "87:\tlearn: 0.0712590\ttotal: 513ms\tremaining: 69.9ms\n",
      "88:\tlearn: 0.0705057\ttotal: 519ms\tremaining: 64.1ms\n",
      "89:\tlearn: 0.0700140\ttotal: 525ms\tremaining: 58.3ms\n",
      "90:\tlearn: 0.0691711\ttotal: 530ms\tremaining: 52.4ms\n",
      "91:\tlearn: 0.0682931\ttotal: 536ms\tremaining: 46.6ms\n",
      "92:\tlearn: 0.0677692\ttotal: 541ms\tremaining: 40.7ms\n",
      "93:\tlearn: 0.0672758\ttotal: 546ms\tremaining: 34.9ms\n",
      "94:\tlearn: 0.0667443\ttotal: 552ms\tremaining: 29.1ms\n",
      "95:\tlearn: 0.0663256\ttotal: 558ms\tremaining: 23.3ms\n",
      "96:\tlearn: 0.0655992\ttotal: 563ms\tremaining: 17.4ms\n",
      "97:\tlearn: 0.0651681\ttotal: 569ms\tremaining: 11.6ms\n",
      "98:\tlearn: 0.0647388\ttotal: 576ms\tremaining: 5.82ms\n",
      "99:\tlearn: 0.0643467\ttotal: 582ms\tremaining: 0us\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0    2.0   3.0\n",
      "0.0  699.0    1.0    7.0   1.0\n",
      "1.0    6.0  466.0    6.0   0.0\n",
      "2.0   11.0    3.0  102.0   2.0\n",
      "3.0    0.0    0.0    5.0  41.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9688888888888889\n",
      "Precision total:  0.9373911316901332\n",
      "Recall total:  0.929473665142466\n",
      "F1 total:  0.9332794760392186\n",
      "BACC total:  0.929473665142466\n",
      "MCC total:  0.9473041295409831\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "bag_cat = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "base_classifier = bag_cat\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_cat'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baggin LGBM\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0    2.0   3.0  4.0\n",
      "0.0  698.0    1.0    7.0   1.0  1.0\n",
      "1.0    7.0  470.0    1.0   0.0  0.0\n",
      "2.0    4.0    1.0  111.0   2.0  0.0\n",
      "3.0    0.0    0.0    3.0  43.0  0.0\n",
      "4.0    0.0    0.0    0.0   0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9792592592592593\n",
      "Precision total:  0.7649733153085713\n",
      "Recall total:  0.7689199758676792\n",
      "F1 total:  0.7668872501126401\n",
      "BACC total:  0.9611499698345991\n",
      "MCC total:  0.964973376002392\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "y_pred contains classes not in y_true\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "base_classifier = lgbm\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lgbm'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0   2.0   3.0\n",
      "0.0  695.0    7.0   5.0   1.0\n",
      "1.0   17.0  454.0   7.0   0.0\n",
      "2.0   14.0    3.0  99.0   2.0\n",
      "3.0    0.0    0.0   8.0  38.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9525925925925925\n",
      "Precision total:  0.9236276481865223\n",
      "Recall total:  0.8991248051068431\n",
      "F1 total:  0.910557358759531\n",
      "BACC total:  0.8991248051068431\n",
      "MCC total:  0.9195604848401054\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "\n",
    "base_classifier = rf\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train_01, y_train_01)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test_01)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_rf'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging with many models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### do bootstrapping "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 1. Multiple subsets are created from the original dataset, selecting observations with replacement.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "\n",
    "num_bootstraps = 10  # Adjust the number of bootstraps as needed\n",
    "\n",
    "original_data_df = X_train_01.assign(label = y_train_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {},
   "outputs": [],
   "source": [
    "boot_df = []\n",
    "for i in range(0,num_bootstraps): \n",
    "    boot_df.append(original_data_df.sample(frac = 1, replace=True).reset_index(drop=True))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rf</th>\n",
       "      <th>ada</th>\n",
       "      <th>xgb</th>\n",
       "      <th>dnn</th>\n",
       "      <th>cat</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.959790</td>\n",
       "      <td>0.315597</td>\n",
       "      <td>0.994228</td>\n",
       "      <td>0.935731</td>\n",
       "      <td>0.995068</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.927451</td>\n",
       "      <td>0.307769</td>\n",
       "      <td>0.996493</td>\n",
       "      <td>0.933363</td>\n",
       "      <td>0.988214</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.543332</td>\n",
       "      <td>0.411966</td>\n",
       "      <td>0.989324</td>\n",
       "      <td>0.471306</td>\n",
       "      <td>0.970823</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.987811</td>\n",
       "      <td>0.303750</td>\n",
       "      <td>0.997760</td>\n",
       "      <td>0.567234</td>\n",
       "      <td>0.995899</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.634661</td>\n",
       "      <td>0.293188</td>\n",
       "      <td>0.884207</td>\n",
       "      <td>0.901184</td>\n",
       "      <td>0.873931</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3145</th>\n",
       "      <td>0.960068</td>\n",
       "      <td>0.319886</td>\n",
       "      <td>0.993254</td>\n",
       "      <td>0.933188</td>\n",
       "      <td>0.994152</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3146</th>\n",
       "      <td>0.960068</td>\n",
       "      <td>0.307373</td>\n",
       "      <td>0.994228</td>\n",
       "      <td>0.937913</td>\n",
       "      <td>0.995631</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3147</th>\n",
       "      <td>0.839586</td>\n",
       "      <td>0.298181</td>\n",
       "      <td>0.943468</td>\n",
       "      <td>0.371196</td>\n",
       "      <td>0.921495</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3148</th>\n",
       "      <td>0.962837</td>\n",
       "      <td>0.319886</td>\n",
       "      <td>0.993577</td>\n",
       "      <td>0.894050</td>\n",
       "      <td>0.994019</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3149</th>\n",
       "      <td>0.944358</td>\n",
       "      <td>0.318544</td>\n",
       "      <td>0.990882</td>\n",
       "      <td>0.933873</td>\n",
       "      <td>0.994279</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3150 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            rf       ada       xgb       dnn       cat  label\n",
       "0     0.959790  0.315597  0.994228  0.935731  0.995068    0.0\n",
       "1     0.927451  0.307769  0.996493  0.933363  0.988214    0.0\n",
       "2     0.543332  0.411966  0.989324  0.471306  0.970823    2.0\n",
       "3     0.987811  0.303750  0.997760  0.567234  0.995899    1.0\n",
       "4     0.634661  0.293188  0.884207  0.901184  0.873931    2.0\n",
       "...        ...       ...       ...       ...       ...    ...\n",
       "3145  0.960068  0.319886  0.993254  0.933188  0.994152    0.0\n",
       "3146  0.960068  0.307373  0.994228  0.937913  0.995631    0.0\n",
       "3147  0.839586  0.298181  0.943468  0.371196  0.921495    0.0\n",
       "3148  0.962837  0.319886  0.993577  0.894050  0.994019    0.0\n",
       "3149  0.944358  0.318544  0.990882  0.933873  0.994279    0.0\n",
       "\n",
       "[3150 rows x 6 columns]"
      ]
     },
     "execution_count": 349,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boot_df[5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.A base model (weak model) is created on each of these subsets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {},
   "outputs": [],
   "source": [
    "bag_comb_pred = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SVM\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "y_train_boot = boot_df[0].pop('label')\n",
    "X_train_boot = boot_df[0]\n",
    "clf.fit(X_train_boot, y_train_boot)\n",
    "preds_svm_01 = clf.predict(X_test_01)\n",
    "bag_comb_pred.append(preds_svm_01)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ADA\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "ada = abc.fit(X_train_01, y_train_01)\n",
    "y_train_boot = boot_df[1].pop('label')\n",
    "X_train_boot = boot_df[1]\n",
    "preds_ada_01 = ada.predict(X_test_01)\n",
    "bag_comb_pred.append(preds_ada_01)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.3353943\ttest: 1.3405362\tbest: 1.3405362 (0)\ttotal: 6.55ms\tremaining: 648ms\n",
      "10:\tlearn: 0.4784180\ttest: 0.4940028\tbest: 0.4940028 (10)\ttotal: 63.1ms\tremaining: 510ms\n",
      "20:\tlearn: 0.2615392\ttest: 0.2800742\tbest: 0.2800742 (20)\ttotal: 122ms\tremaining: 460ms\n",
      "30:\tlearn: 0.1744494\ttest: 0.1978095\tbest: 0.1978095 (30)\ttotal: 180ms\tremaining: 401ms\n",
      "40:\tlearn: 0.1355911\ttest: 0.1604291\tbest: 0.1604291 (40)\ttotal: 236ms\tremaining: 339ms\n",
      "50:\tlearn: 0.1118923\ttest: 0.1415299\tbest: 0.1415299 (50)\ttotal: 285ms\tremaining: 274ms\n",
      "60:\tlearn: 0.0967557\ttest: 0.1291668\tbest: 0.1291668 (60)\ttotal: 339ms\tremaining: 217ms\n",
      "70:\tlearn: 0.0847308\ttest: 0.1196154\tbest: 0.1196154 (70)\ttotal: 392ms\tremaining: 160ms\n",
      "80:\tlearn: 0.0757855\ttest: 0.1120742\tbest: 0.1120742 (80)\ttotal: 441ms\tremaining: 103ms\n",
      "90:\tlearn: 0.0696515\ttest: 0.1070302\tbest: 0.1070302 (90)\ttotal: 490ms\tremaining: 48.4ms\n",
      "99:\tlearn: 0.0640921\ttest: 0.1031307\tbest: 0.1031307 (99)\ttotal: 535ms\tremaining: 0us\n",
      "\n",
      "bestTest = 0.1031306785\n",
      "bestIteration = 99\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Catboost\n",
    "\n",
    "cat_01 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "y_train_boot = boot_df[2].pop('label')\n",
    "X_train_boot = boot_df[2]\n",
    "cat_01.fit(X_train_boot, y_train_boot, eval_set=(X_test_01, y_test_01), verbose=10)\n",
    "preds_cat = cat_01.predict(X_test_01)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "pred_label = preds_cat\n",
    "bag_comb_pred.append(preds_cat)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    }
   ],
   "source": [
    "#MLP\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "y_train_boot = boot_df[3].pop('label')\n",
    "X_train_boot = boot_df[3]\n",
    "if 1 == 1 and 0 == 0:\n",
    "    MLP = mlp.fit(X_train_boot, y_train_boot)\n",
    "    y_pred = MLP.predict_proba(X_test_01)\n",
    "    preds_mlp_01 = np.argmax(y_pred,axis = 1)\n",
    "\n",
    "bag_comb_pred.append(preds_mlp_01)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LGBM\n",
    "lgbm = LGBMClassifier()\n",
    "y_train_boot = boot_df[4].pop('label')\n",
    "X_train_boot = boot_df[4]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    lgbm.fit(X_train_boot, y_train_boot)\n",
    "    preds_lgbm_01 = lgbm.predict(X_test_01)\n",
    "    bag_comb_pred.append(preds_lgbm_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [],
   "source": [
    "#KNN\n",
    "knn_clf_01=KNeighborsClassifier(n_neighbors = 5)\n",
    "y_train_boot = boot_df[5].pop('label')\n",
    "X_train_boot = boot_df[5]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    knn_clf_01.fit(X_train_boot,y_train_boot)\n",
    "if use_model_knn == 1:\n",
    "    preds_knn =knn_clf_01.predict(X_test_01)\n",
    "    bag_comb_pred.append(preds_knn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Random Forest\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "y_train_boot = boot_df[6].pop('label')\n",
    "X_train_boot = boot_df[6]\n",
    "\n",
    "if True == True:\n",
    "    model_rf_01 = rf.fit(X_train_boot,y_train_boot)\n",
    "    preds_rf_01 = model_rf_01.predict(X_test_01)\n",
    "    bag_comb_pred.append(preds_rf_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "20/20 [==============================] - 0s 6ms/step - loss: 1.5351 - accuracy: 0.4948 - val_loss: 1.4488 - val_accuracy: 0.5571\n",
      "Epoch 2/100\n",
      "20/20 [==============================] - 0s 2ms/step - loss: 1.4116 - accuracy: 0.5143 - val_loss: 1.3324 - val_accuracy: 0.5571\n",
      "Epoch 3/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.3051 - accuracy: 0.5083 - val_loss: 1.2051 - val_accuracy: 0.5571\n",
      "Epoch 4/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.1815 - accuracy: 0.5266 - val_loss: 1.0774 - val_accuracy: 0.5571\n",
      "Epoch 5/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.0940 - accuracy: 0.5210 - val_loss: 0.9821 - val_accuracy: 0.5571\n",
      "Epoch 6/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.0441 - accuracy: 0.5107 - val_loss: 0.9375 - val_accuracy: 0.5571\n",
      "Epoch 7/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.0274 - accuracy: 0.5194 - val_loss: 0.9203 - val_accuracy: 0.5571\n",
      "Epoch 8/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 1.0150 - accuracy: 0.5262 - val_loss: 0.9140 - val_accuracy: 0.5571\n",
      "Epoch 9/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.0180 - accuracy: 0.5167 - val_loss: 0.9119 - val_accuracy: 0.5571\n",
      "Epoch 10/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9942 - accuracy: 0.5258 - val_loss: 0.9077 - val_accuracy: 0.5571\n",
      "Epoch 11/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.9979 - accuracy: 0.5198 - val_loss: 0.9041 - val_accuracy: 0.5587\n",
      "Epoch 12/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.9830 - accuracy: 0.5278 - val_loss: 0.8993 - val_accuracy: 0.5603\n",
      "Epoch 13/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9790 - accuracy: 0.5254 - val_loss: 0.8953 - val_accuracy: 0.5603\n",
      "Epoch 14/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9720 - accuracy: 0.5222 - val_loss: 0.8892 - val_accuracy: 0.5603\n",
      "Epoch 15/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.9638 - accuracy: 0.5274 - val_loss: 0.8835 - val_accuracy: 0.5619\n",
      "Epoch 16/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.9582 - accuracy: 0.5321 - val_loss: 0.8773 - val_accuracy: 0.5683\n",
      "Epoch 17/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9393 - accuracy: 0.5313 - val_loss: 0.8714 - val_accuracy: 0.5730\n",
      "Epoch 18/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.9393 - accuracy: 0.5286 - val_loss: 0.8643 - val_accuracy: 0.5714\n",
      "Epoch 19/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.9370 - accuracy: 0.5290 - val_loss: 0.8589 - val_accuracy: 0.5746\n",
      "Epoch 20/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.9279 - accuracy: 0.5290 - val_loss: 0.8517 - val_accuracy: 0.5746\n",
      "Epoch 21/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.9232 - accuracy: 0.5345 - val_loss: 0.8431 - val_accuracy: 0.5762\n",
      "Epoch 22/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.9178 - accuracy: 0.5258 - val_loss: 0.8357 - val_accuracy: 0.5730\n",
      "Epoch 23/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.9032 - accuracy: 0.5282 - val_loss: 0.8276 - val_accuracy: 0.5698\n",
      "Epoch 24/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.8867 - accuracy: 0.5448 - val_loss: 0.8203 - val_accuracy: 0.5619\n",
      "Epoch 25/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.8787 - accuracy: 0.5369 - val_loss: 0.8118 - val_accuracy: 0.5603\n",
      "Epoch 26/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.8741 - accuracy: 0.5333 - val_loss: 0.8049 - val_accuracy: 0.5587\n",
      "Epoch 27/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.8657 - accuracy: 0.5329 - val_loss: 0.8010 - val_accuracy: 0.5603\n",
      "Epoch 28/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.8604 - accuracy: 0.5385 - val_loss: 0.7944 - val_accuracy: 0.5603\n",
      "Epoch 29/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.8477 - accuracy: 0.5496 - val_loss: 0.7873 - val_accuracy: 0.5635\n",
      "Epoch 30/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.8387 - accuracy: 0.5512 - val_loss: 0.7811 - val_accuracy: 0.5603\n",
      "Epoch 31/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.8318 - accuracy: 0.5540 - val_loss: 0.7714 - val_accuracy: 0.5635\n"
     ]
    }
   ],
   "source": [
    "#DNN\n",
    "#Model Parameters\n",
    "y_train_boot = boot_df[7].pop('label')\n",
    "X_train_boot = boot_df[7]\n",
    "\n",
    "\n",
    "dropout_rate = 0.02\n",
    "nodes = 3\n",
    "out_layer = 5\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "num_columns = X_train_boot.shape[1]\n",
    "dnn_01 = tf.keras.Sequential()\n",
    "# Input layer\n",
    "dnn_01.add(tf.keras.Input(shape=(num_columns,)))\n",
    "# Dense layers with dropout\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "# Output layer\n",
    "dnn_01.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "dnn_01.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "from keras.callbacks import EarlyStopping\n",
    "# Define EarlyStopping callback\n",
    "early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "dnn_01.fit(X_train_boot, y_train_boot, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "pred_dnn = dnn_01.predict(X_test_01)\n",
    "preds_dnn_01 = np.argmax(pred_dnn,axis = 1)\n",
    "bag_comb_pred.append(preds_dnn_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    }
   ],
   "source": [
    "#LogReg\n",
    "logreg_01 = LogisticRegression()\n",
    "y_train_boot = boot_df[8].pop('label')\n",
    "X_train_boot = boot_df[8]\n",
    "\n",
    "logreg_01.fit(X_train_boot,y_train_boot)\n",
    "preds_logreg =logreg_01.predict(X_test_01)\n",
    "bag_comb_pred.append(preds_logreg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_boot = boot_df[9].pop('label')\n",
    "X_train_boot = boot_df[9]\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train_boot, label=y_train_boot)\n",
    "dtest = xgb.DMatrix(X_test_01, label=y_test_01)\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 5,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_01 = xgb.train(params, dtrain, num_round)\n",
    "preds_xgb_01 = xgb_01.predict(dtest)\n",
    "bag_comb_pred.append(preds_xgb_01)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. The models run in parallel and are independent of each other."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0         0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "1         0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "2         1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "3         2.0      2.0      2.0        2      2.0      2.0      2.0        1   \n",
      "4         0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "...       ...      ...      ...      ...      ...      ...      ...      ...   \n",
      "1345      1.0      1.0      0.0        1      0.0      0.0      0.0        0   \n",
      "1346      1.0      3.0      3.0        3      3.0      3.0      3.0        1   \n",
      "1347      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "1348      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "1349      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "\n",
      "      model_8  model_9  \n",
      "0         0.0      0.0  \n",
      "1         0.0      0.0  \n",
      "2         1.0      1.0  \n",
      "3         2.0      2.0  \n",
      "4         0.0      0.0  \n",
      "...       ...      ...  \n",
      "1345      1.0      0.0  \n",
      "1346      1.0      3.0  \n",
      "1347      1.0      1.0  \n",
      "1348      0.0      0.0  \n",
      "1349      0.0      0.0  \n",
      "\n",
      "[1350 rows x 10 columns]\n"
     ]
    }
   ],
   "source": [
    "bag_vot_df = pd.DataFrame()\n",
    "for i in range(0,len(bag_comb_pred)):\n",
    "    bag_vot_df[f'model_{i}'] =  bag_comb_pred[i]\n",
    "print(bag_vot_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0         0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "1         0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "2         1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "3         2.0      2.0      2.0        2      2.0      2.0      2.0        1   \n",
      "4         0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "...       ...      ...      ...      ...      ...      ...      ...      ...   \n",
      "1345      1.0      1.0      0.0        1      0.0      0.0      0.0        0   \n",
      "1346      1.0      3.0      3.0        3      3.0      3.0      3.0        1   \n",
      "1347      1.0      1.0      1.0        1      1.0      1.0      1.0        0   \n",
      "1348      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "1349      0.0      0.0      0.0        0      0.0      0.0      0.0        0   \n",
      "\n",
      "      model_8  model_9  ensemble  \n",
      "0         0.0      0.0         0  \n",
      "1         0.0      0.0         0  \n",
      "2         1.0      1.0         1  \n",
      "3         2.0      2.0         2  \n",
      "4         0.0      0.0         0  \n",
      "...       ...      ...       ...  \n",
      "1345      1.0      0.0         0  \n",
      "1346      1.0      3.0         3  \n",
      "1347      1.0      1.0         1  \n",
      "1348      0.0      0.0         0  \n",
      "1349      0.0      0.0         0  \n",
      "\n",
      "[1350 rows x 11 columns]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0       0\n",
       "1       0\n",
       "2       1\n",
       "3       2\n",
       "4       0\n",
       "       ..\n",
       "1345    0\n",
       "1346    3\n",
       "1347    1\n",
       "1348    0\n",
       "1349    0\n",
       "Name: ensemble, Length: 1350, dtype: int64"
      ]
     },
     "execution_count": 362,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Voting start\n",
    "\n",
    "predictions = bag_vot_df \n",
    "\n",
    "# Use the mode function along axis 1 to get the most common prediction for each row\n",
    "ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "# Add the ensemble predictions to the DataFrame\n",
    "bag_vot_df['ensemble'] = ensemble_predictions.astype(int)\n",
    "\n",
    "# Display the DataFrame with ensemble predictions\n",
    "print(bag_vot_df)\n",
    "\n",
    "pred_label = bag_vot_df ['ensemble'].values\n",
    "bag_vot_df.pop('ensemble')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0      1     2     3    4\n",
      "0  698.0    3.0   5.0   1.0  1.0\n",
      "1    7.0  468.0   3.0   0.0  0.0\n",
      "2   18.0   24.0  74.0   2.0  0.0\n",
      "3    0.0    2.0   3.0  41.0  0.0\n",
      "4    0.0    0.0   0.0   0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9488888888888889\n",
      "Precision total:  0.7418956339794672\n",
      "Recall total:  0.6966756392033046\n",
      "F1 total:  0.7151433459670284\n",
      "BACC total:  0.8708445490041307\n",
      "MCC total:  0.913039250261179\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "y_pred contains classes not in y_true\n"
     ]
    }
   ],
   "source": [
    "name='bag_comb'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining DNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "Model: \"sequential_11\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_66 (Dense)             (None, 3)                 18        \n",
      "_________________________________________________________________\n",
      "dropout_55 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_67 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_56 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_68 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_57 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_69 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_58 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_70 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_59 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_71 (Dense)             (None, 5)                 20        \n",
      "=================================================================\n",
      "Total params: 86\n",
      "Trainable params: 86\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining DNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_dnn = time.time()\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "#Model Parameters\n",
    "dropout_rate = 0.2\n",
    "nodes = 3\n",
    "out_layer = 5\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "\n",
    "\n",
    "num_columns = X_train_01.shape[1]\n",
    "\n",
    "dnn_01 = tf.keras.Sequential()\n",
    "\n",
    "# Input layer\n",
    "dnn_01.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "# Dense layers with dropout\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn_01.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_01.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# Output layer\n",
    "dnn_01.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "\n",
    "dnn_01.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "\n",
    "dnn_01.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Training DNN\n",
      "---------------------------------------------------------------------------------\n",
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20/20 [==============================] - 1s 6ms/step - loss: 1.6898 - accuracy: 0.2079 - val_loss: 1.6016 - val_accuracy: 0.3381\n",
      "Epoch 2/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.5733 - accuracy: 0.3417 - val_loss: 1.5267 - val_accuracy: 0.3349\n",
      "Epoch 3/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.5059 - accuracy: 0.3746 - val_loss: 1.4609 - val_accuracy: 0.3556\n",
      "Epoch 4/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.4414 - accuracy: 0.4159 - val_loss: 1.3946 - val_accuracy: 0.3540\n",
      "Epoch 5/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 1.3699 - accuracy: 0.4337 - val_loss: 1.3274 - val_accuracy: 0.3444\n",
      "Epoch 6/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.3151 - accuracy: 0.4278 - val_loss: 1.2583 - val_accuracy: 0.2413\n",
      "Epoch 7/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 1.2605 - accuracy: 0.4321 - val_loss: 1.1955 - val_accuracy: 0.5127\n",
      "Epoch 8/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 1.2087 - accuracy: 0.4611 - val_loss: 1.1433 - val_accuracy: 0.5127\n",
      "Epoch 9/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 1.1693 - accuracy: 0.4738 - val_loss: 1.1017 - val_accuracy: 0.5127\n",
      "Epoch 10/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 1.1376 - accuracy: 0.4861 - val_loss: 1.0719 - val_accuracy: 0.5127\n",
      "Epoch 11/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 1.1102 - accuracy: 0.4873 - val_loss: 1.0546 - val_accuracy: 0.5127\n",
      "Epoch 12/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.0987 - accuracy: 0.4905 - val_loss: 1.0378 - val_accuracy: 0.5127\n",
      "Epoch 13/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.0910 - accuracy: 0.4857 - val_loss: 1.0256 - val_accuracy: 0.5127\n",
      "Epoch 14/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.0663 - accuracy: 0.4810 - val_loss: 1.0125 - val_accuracy: 0.5127\n",
      "Epoch 15/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 1.0595 - accuracy: 0.4929 - val_loss: 0.9986 - val_accuracy: 0.5127\n",
      "Epoch 16/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 1.0414 - accuracy: 0.4988 - val_loss: 0.9834 - val_accuracy: 0.5127\n",
      "Epoch 17/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 1.0283 - accuracy: 0.5012 - val_loss: 0.9696 - val_accuracy: 0.5143\n",
      "Epoch 18/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 1.0219 - accuracy: 0.4952 - val_loss: 0.9578 - val_accuracy: 0.5143\n",
      "Epoch 19/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 1.0178 - accuracy: 0.4988 - val_loss: 0.9477 - val_accuracy: 0.5175\n",
      "Epoch 20/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9982 - accuracy: 0.5071 - val_loss: 0.9412 - val_accuracy: 0.5254\n",
      "Epoch 21/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9983 - accuracy: 0.5107 - val_loss: 0.9347 - val_accuracy: 0.5286\n",
      "Epoch 22/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 1.0058 - accuracy: 0.5000 - val_loss: 0.9302 - val_accuracy: 0.5286\n",
      "Epoch 23/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9885 - accuracy: 0.5115 - val_loss: 0.9267 - val_accuracy: 0.5286\n",
      "Epoch 24/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9872 - accuracy: 0.5179 - val_loss: 0.9215 - val_accuracy: 0.5286\n",
      "Epoch 25/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9871 - accuracy: 0.5413 - val_loss: 0.9198 - val_accuracy: 0.5286\n",
      "Epoch 26/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9869 - accuracy: 0.5317 - val_loss: 0.9160 - val_accuracy: 0.5302\n",
      "Epoch 27/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9742 - accuracy: 0.5413 - val_loss: 0.9142 - val_accuracy: 0.5286\n",
      "Epoch 28/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9726 - accuracy: 0.5389 - val_loss: 0.9111 - val_accuracy: 0.5317\n",
      "Epoch 29/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9911 - accuracy: 0.5310 - val_loss: 0.9085 - val_accuracy: 0.5302\n",
      "Epoch 30/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9692 - accuracy: 0.5365 - val_loss: 0.9064 - val_accuracy: 0.5302\n",
      "Epoch 31/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9642 - accuracy: 0.5385 - val_loss: 0.9036 - val_accuracy: 0.5317\n",
      "Epoch 32/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9606 - accuracy: 0.5460 - val_loss: 0.9015 - val_accuracy: 0.5333\n",
      "Epoch 33/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9719 - accuracy: 0.5377 - val_loss: 0.9014 - val_accuracy: 0.5349\n",
      "Epoch 34/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9697 - accuracy: 0.5377 - val_loss: 0.8987 - val_accuracy: 0.5349\n",
      "Epoch 35/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.9526 - accuracy: 0.5456 - val_loss: 0.8945 - val_accuracy: 0.5365\n",
      "Epoch 36/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9571 - accuracy: 0.5448 - val_loss: 0.8911 - val_accuracy: 0.5365\n",
      "Epoch 37/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9436 - accuracy: 0.5480 - val_loss: 0.8888 - val_accuracy: 0.5365\n",
      "Epoch 38/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9509 - accuracy: 0.5452 - val_loss: 0.8889 - val_accuracy: 0.5397\n",
      "Epoch 39/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9609 - accuracy: 0.5425 - val_loss: 0.8903 - val_accuracy: 0.5429\n",
      "Epoch 40/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9405 - accuracy: 0.5512 - val_loss: 0.8874 - val_accuracy: 0.5397\n",
      "Epoch 41/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.9525 - accuracy: 0.5480 - val_loss: 0.8868 - val_accuracy: 0.5429\n",
      "Epoch 42/100\n",
      "20/20 [==============================] - 0s 3ms/step - loss: 0.9489 - accuracy: 0.5452 - val_loss: 0.8873 - val_accuracy: 0.5429\n",
      "Epoch 43/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9415 - accuracy: 0.5437 - val_loss: 0.8848 - val_accuracy: 0.5429\n",
      "Epoch 44/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9413 - accuracy: 0.5409 - val_loss: 0.8831 - val_accuracy: 0.5429\n",
      "Epoch 45/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9382 - accuracy: 0.5536 - val_loss: 0.8831 - val_accuracy: 0.5429\n",
      "Epoch 46/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9358 - accuracy: 0.5437 - val_loss: 0.8802 - val_accuracy: 0.5429\n",
      "Epoch 47/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9392 - accuracy: 0.5365 - val_loss: 0.8800 - val_accuracy: 0.5429\n",
      "Epoch 48/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9232 - accuracy: 0.5532 - val_loss: 0.8763 - val_accuracy: 0.5429\n",
      "Epoch 49/100\n",
      "20/20 [==============================] - 0s 4ms/step - loss: 0.9455 - accuracy: 0.5492 - val_loss: 0.8756 - val_accuracy: 0.5429\n"
     ]
    }
   ],
   "source": [
    "#DNN\n",
    "try:\n",
    "    from keras.callbacks import EarlyStopping\n",
    "\n",
    "    # Define EarlyStopping callback\n",
    "    early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training DNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training DNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    # Convert Y_test back to its original format\n",
    "    # y_test = np.argmax(Y_test, axis=1)\n",
    "\n",
    "    # Start the timer\n",
    "    start = time.time()\n",
    "    dnn_01.fit(X_train_01, y_train_01, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "    # End the timer\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "\n",
    "except: \n",
    "    None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DNN\n",
    "try:\n",
    "    start = time.time()\n",
    "    pred_dnn = dnn_01.predict(X_test_01)\n",
    "    preds_dnn_01 = np.argmax(pred_dnn,axis = 1)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "except:\n",
    "        with open(output_file_name, \"a\") as f: print('error', file = f)\n",
    "        preds_dnn_01 = 0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0   1.0   2.0  3.0\n",
      "0.0  705.0   3.0   0.0  0.0\n",
      "1.0  454.0  15.0   9.0  0.0\n",
      "2.0   94.0   2.0  22.0  0.0\n",
      "3.0    3.0   5.0  38.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.5614814814814815\n",
      "Precision total:  0.428007592541309\n",
      "Recall total:  0.4633076053810553\n",
      "F1 total:  0.35960857981483274\n",
      "BACC total:  0.4633076053810553\n",
      "MCC total:  0.23415900890952782\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    name = 'dnn'\n",
    "    pred_label = preds_dnn_01\n",
    "        \n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_dnn\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n",
    "except: None    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining SVM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#SVM\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining SVM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_svm = time.time()\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "#SVM\n",
    "start = time.time()\n",
    "clf.fit(X_train_01, y_train_01)\n",
    "end = time.time()\n",
    "clf.score(X_train_01, y_train_01)\n",
    "time_taken = end - start\n",
    "with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "joblib.dump(clf, 'svm_level_01.joblib')\n",
    "\n",
    "\n",
    "clf = loaded_model = joblib.load('svm_level_01.joblib')\n",
    "\n",
    "\n",
    "#SVM\n",
    "start = time.time()\n",
    "preds_svm_01 = clf.predict(X_test_01)\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0   2.0   3.0\n",
      "0.0  608.0   90.0  10.0   0.0\n",
      "1.0    7.0  454.0  16.0   1.0\n",
      "2.0   19.0   52.0  44.0   3.0\n",
      "3.0    2.0   14.0   7.0  23.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8362962962962963\n",
      "Precision total:  0.7808793902824216\n",
      "Recall total:  0.6703573032645439\n",
      "F1 total:  0.7051849414686844\n",
      "BACC total:  0.6703573032645439\n",
      "MCC total:  0.7297640488769191\n"
     ]
    }
   ],
   "source": [
    "\n",
    "pred_label = preds_svm_01\n",
    "name = 'svm'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start_svm\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining RF Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0   2.0   3.0\n",
      "0.0  696.0    8.0   3.0   1.0\n",
      "1.0   22.0  456.0   0.0   0.0\n",
      "2.0   20.0   19.0  78.0   1.0\n",
      "3.0    0.0    1.0   8.0  37.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9385185185185185\n",
      "Precision total:  0.9275901585812147\n",
      "Recall total:  0.8505976295236539\n",
      "F1 total:  0.8837229933461535\n",
      "BACC total:  0.8505976295236539\n",
      "MCC total:  0.8948821497042864\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining RF Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_rf = time.time()\n",
    "\n",
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "#------------------------------------------------------------------------------\n",
    "\n",
    "if True == True:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Training RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    model_rf_01 = rf.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(model_rf_01, 'rf_base_model_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    model_rf_01  = joblib.load('rf_base_model_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    preds_rf_01 = model_rf_01.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('-------------------------------------------------------', file = f)\n",
    "pred_label = preds_rf_01\n",
    "name='rf'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start_rf\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining LGBM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LGBM\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction LGBM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0   2.0   3.0  4.0\n",
      "0.0  670.0   16.0  20.0   2.0  0.0\n",
      "1.0   15.0  458.0   4.0   1.0  0.0\n",
      "2.0    7.0   38.0  67.0   3.0  3.0\n",
      "3.0    0.0    2.0   7.0  36.0  1.0\n",
      "4.0    0.0    0.0   0.0   0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9118518518518518\n",
      "Precision total:  0.6800150005347498\n",
      "Recall total:  0.6509783970506768\n",
      "F1 total:  0.6638164284938479\n",
      "BACC total:  0.8137229963133461\n",
      "MCC total:  0.8511255608397971\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Recall is ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.\n",
      "y_pred contains classes not in y_true\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining LGBM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#LGBM\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "start_lgbm = time.time()\n",
    "\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    lgbm.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(lgbm, 'lgbm_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    lgbm = joblib.load('lgbm_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('Prediction LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #LGBM\n",
    "    start = time.time()\n",
    "    preds_lgbm_01 = lgbm.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    pred_label = preds_lgbm_01\n",
    "    name='lgbm'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_lgbm\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining MLP Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training MLP\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0      1     2     3\n",
      "0  638.0   46.0  24.0   0.0\n",
      "1    7.0  448.0  13.0  10.0\n",
      "2   18.0   47.0  48.0   5.0\n",
      "3    0.0    0.0   1.0  45.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8733333333333333\n",
      "Precision total:  0.7746320656336498\n",
      "Recall total:  0.8058522419522102\n",
      "F1 total:  0.7824114441896266\n",
      "BACC total:  0.8058522419522102\n",
      "MCC total:  0.7891966676064138\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#MLP\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining MLP Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_mlp = time.time()\n",
    "\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "import time\n",
    "\n",
    "# create MLPClassifier instance\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training MLP')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Training MLP', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    start = time.time()\n",
    "    MLP = mlp.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(MLP, 'mlp_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    MLP = joblib.load('mlp_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    #MLP\n",
    "    start = time.time()\n",
    "    y_pred = MLP.predict_proba(X_test_01)\n",
    "    preds_mlp_01 = np.argmax(y_pred,axis = 1)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#MLP\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('MLP 01 model', file = f)\n",
    "    pred_label = preds_mlp_01\n",
    "    name='mlp'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_mlp\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ADA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining ADA Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training ADA\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction ADA\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0   2.0   3.0\n",
      "0.0  614.0   87.0   5.0   2.0\n",
      "1.0    1.0  465.0  12.0   0.0\n",
      "2.0   23.0   70.0  22.0   3.0\n",
      "3.0    0.0    0.0   4.0  42.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8466666666666667\n",
      "Precision total:  0.7788039494579262\n",
      "Recall total:  0.7348797854813462\n",
      "F1 total:  0.7335762788526575\n",
      "BACC total:  0.7348797854813462\n",
      "MCC total:  0.7491215903703766\n"
     ]
    }
   ],
   "source": [
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining ADA Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#ADA\n",
    "start_ada = time.time()\n",
    "\n",
    "\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "import time\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "\n",
    "    start = time.time()\n",
    "    ada = abc.fit(X_train_01, y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "\n",
    "    # Assuming 'model' is your trained model\n",
    "    joblib.dump(ada, 'ada_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "    ada = joblib.load('ada_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    print('Prediction ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "    start = time.time()\n",
    "    preds_ada_01 = ada.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('ADA 01 model', file = f)\n",
    "\n",
    "\n",
    "    pred_label = preds_ada_01\n",
    "    name='ada'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_ada\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining KNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training KNN\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0   2.0   3.0\n",
      "0.0  688.0    5.0  15.0   0.0\n",
      "1.0   12.0  454.0  11.0   1.0\n",
      "2.0   13.0    6.0  91.0   8.0\n",
      "3.0    0.0    0.0   5.0  41.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9437037037037037\n",
      "Precision total:  0.8767956529403199\n",
      "Recall total:  0.8960082489781278\n",
      "F1 total:  0.8859291429757993\n",
      "BACC total:  0.8960082489781278\n",
      "MCC total:  0.9049843664792343\n"
     ]
    }
   ],
   "source": [
    "#KNN\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining KNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "start_knn = time.time()\n",
    "\n",
    "knn_clf_01=KNeighborsClassifier(n_neighbors = 5)\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training KNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training KNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    knn_clf_01.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(knn_clf_01, 'knn_01.joblib')\n",
    "\n",
    "\n",
    "if load_model_knn == 1:\n",
    "    knn_clf_01 = joblib.load('knn_01.joblib')\n",
    "\n",
    "if use_model_knn == 1:\n",
    "\n",
    "    #KNN\n",
    "    start = time.time()\n",
    "    preds_knn =knn_clf_01.predict(X_test_01)\n",
    "    preds_knn\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('KNN 01 model', file = f)\n",
    "\n",
    "    pred_label = preds_knn\n",
    "    name='knn'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start_knn\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Log Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LR \n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0   2.0   3.0\n",
      "0.0  625.0   75.0   6.0   2.0\n",
      "1.0   12.0  447.0  15.0   4.0\n",
      "2.0   24.0   49.0  40.0   5.0\n",
      "3.0    0.0    5.0   0.0  41.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8540740740740741\n",
      "Precision total:  0.791444493774797\n",
      "Recall total:  0.7620505509425275\n",
      "F1 total:  0.7612336622580748\n",
      "BACC total:  0.7620505509425275\n",
      "MCC total:  0.7567670358085511\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_01 = LogisticRegression()\n",
    "start_lr = time.time()\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LR ')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LR', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    logreg_01.fit(X_train_01,y_train_01)\n",
    "    end = time.time()\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(logreg_01, 'logreg_01.joblib')\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "    logreg_01 = joblib.load('logreg_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    #lR\n",
    "    start = time.time()\n",
    "    preds_logreg =logreg_01.predict(X_test_01)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#LR\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('LR 01 model', file = f)\n",
    "\n",
    "    pred_label = preds_logreg\n",
    "    # pred_label = label[ypred]\n",
    "    name='lr'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_lr\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.3377869\ttest: 1.3409543\tbest: 1.3409543 (0)\ttotal: 6.83ms\tremaining: 676ms\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10:\tlearn: 0.4872332\ttest: 0.4953866\tbest: 0.4953866 (10)\ttotal: 63.1ms\tremaining: 511ms\n",
      "20:\tlearn: 0.2735539\ttest: 0.2851215\tbest: 0.2851215 (20)\ttotal: 126ms\tremaining: 475ms\n",
      "30:\tlearn: 0.1858267\ttest: 0.2003851\tbest: 0.2003851 (30)\ttotal: 193ms\tremaining: 429ms\n",
      "40:\tlearn: 0.1451702\ttest: 0.1600026\tbest: 0.1600026 (40)\ttotal: 259ms\tremaining: 373ms\n",
      "50:\tlearn: 0.1219310\ttest: 0.1366431\tbest: 0.1366431 (50)\ttotal: 319ms\tremaining: 307ms\n",
      "60:\tlearn: 0.1072353\ttest: 0.1231615\tbest: 0.1231615 (60)\ttotal: 373ms\tremaining: 239ms\n",
      "70:\tlearn: 0.0976357\ttest: 0.1141141\tbest: 0.1141141 (70)\ttotal: 426ms\tremaining: 174ms\n",
      "80:\tlearn: 0.0893530\ttest: 0.1061239\tbest: 0.1061239 (80)\ttotal: 490ms\tremaining: 115ms\n",
      "90:\tlearn: 0.0826111\ttest: 0.1007402\tbest: 0.1007402 (90)\ttotal: 554ms\tremaining: 54.8ms\n",
      "99:\tlearn: 0.0779486\ttest: 0.0967726\tbest: 0.0967726 (99)\ttotal: 611ms\tremaining: 0us\n",
      "\n",
      "bestTest = 0.09677261405\n",
      "bestIteration = 99\n",
      "\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0    2.0   3.0\n",
      "0.0  699.0    2.0    6.0   1.0\n",
      "1.0    5.0  472.0    1.0   0.0\n",
      "2.0   10.0    3.0  103.0   2.0\n",
      "3.0    0.0    0.0    4.0  42.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9748148148148148\n",
      "Precision total:  0.951337880402078\n",
      "Recall total:  0.9401651671327658\n",
      "F1 total:  0.9456529989553877\n",
      "BACC total:  0.9401651671327658\n",
      "MCC total:  0.9572696145611932\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "cat_01 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "# Fit the model\n",
    "cat_01.fit(X_train_01, y_train_01, eval_set=(X_test_01, y_test_01), verbose=10)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_cat = cat_01.predict(X_test_01)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('catboost', file = f)\n",
    "\n",
    "\n",
    "pred_label = preds_cat\n",
    "name='cat'\n",
    "metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_01\"] = Acc\n",
    "globals()[f\"{name}_pre_01\"] = Precision\n",
    "globals()[f\"{name}_rec_01\"] = Recall\n",
    "globals()[f\"{name}_f1_01\"] = F1\n",
    "globals()[f\"{name}_bacc_01\"] = BACC\n",
    "globals()[f\"{name}_mcc_01\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "       0.0    1.0    2.0   3.0\n",
      "0.0  698.0    2.0    7.0   1.0\n",
      "1.0    6.0  468.0    4.0   0.0\n",
      "2.0    5.0    6.0  106.0   1.0\n",
      "3.0    0.0    0.0    5.0  41.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9725925925925926\n",
      "Precision total:  0.9475048247073419\n",
      "Recall total:  0.9386411591736221\n",
      "F1 total:  0.9427484202671987\n",
      "BACC total:  0.9386411591736221\n",
      "MCC total:  0.953618349169585\n"
     ]
    }
   ],
   "source": [
    "\n",
    "start = time.time()\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train_01, label=y_train_01)\n",
    "dtest = xgb.DMatrix(X_test_01, label=y_test_01)\n",
    "\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 5,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_01 = xgb.train(params, dtrain, num_round)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_xgb_01 = xgb_01.predict(dtest)\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('xgboost base model', file = f)\n",
    "\n",
    "    pred_label = preds_xgb_01\n",
    "    name='xgb'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test_01)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "\n",
    "    globals()[f\"{name}_acc_01\"] = Acc\n",
    "    globals()[f\"{name}_pre_01\"] = Precision\n",
    "    globals()[f\"{name}_rec_01\"] = Recall\n",
    "    globals()[f\"{name}_f1_01\"] = F1\n",
    "    globals()[f\"{name}_bacc_01\"] = BACC\n",
    "    globals()[f\"{name}_mcc_01\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_01\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " ### Generating Summary Metric Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+----------+----------+----------+----------+\n",
      "| Models      |   ACC-01 |   PRE-01 |   REC-01 |    F1-01 |\n",
      "+=============+==========+==========+==========+==========+\n",
      "| CAT         | 0.974815 | 0.951338 | 0.940165 | 0.945653 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| XGB         | 0.972593 | 0.947505 | 0.938641 | 0.942748 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| Bag_cat     | 0.968889 | 0.937391 | 0.929474 | 0.933279 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| Bag_rf      | 0.952593 | 0.923628 | 0.899125 | 0.910557 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| KNN         | 0.943704 | 0.876796 | 0.896008 | 0.885929 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| RF          | 0.938519 | 0.92759  | 0.850598 | 0.883723 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| Bag_knn     | 0.942963 | 0.866968 | 0.901993 | 0.883049 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| MLP         | 0.873333 | 0.774632 | 0.805852 | 0.782411 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| Bag_lgbm    | 0.979259 | 0.764973 | 0.76892  | 0.766887 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| Bag_DT      | 0.975556 | 0.761863 | 0.764819 | 0.763201 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| LR          | 0.854074 | 0.791444 | 0.762051 | 0.761234 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| Bag_LR      | 0.853333 | 0.782177 | 0.773072 | 0.760543 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| ADA         | 0.846667 | 0.778804 | 0.73488  | 0.733576 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| Bag_comb    | 0.948889 | 0.741896 | 0.696676 | 0.715143 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| SVM         | 0.836296 | 0.780879 | 0.670357 | 0.705185 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| Bag_svm     | 0.836296 | 0.762437 | 0.687209 | 0.69673  |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| LGBM        | 0.911852 | 0.680015 | 0.650978 | 0.663816 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| Bag_mlp     | 0.87037  | 0.632964 | 0.650034 | 0.632535 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| Bag_ada     | 0.571111 | 0.584733 | 0.53065  | 0.520143 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| DNN         | 0.561481 | 0.428008 | 0.463308 | 0.359609 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| weighed_avg | 0.353333 | 0.132868 | 0.246419 | 0.144717 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| avg         | 0.354815 | 0.12729  | 0.249164 | 0.138068 |\n",
      "+-------------+----------+----------+----------+----------+\n",
      "| VOTING      | 0        | 0        | 0        | 0        |\n",
      "+-------------+----------+----------+----------+----------+\n"
     ]
    }
   ],
   "source": [
    "\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "                'avg',\n",
    "                'weighed_avg'\n",
    "                ]\n",
    "\n",
    "data = [[\"\" for _ in range(5)] for _ in range(len(names_models))]\n",
    "\n",
    "level_01_acc = [\n",
    "                ada_acc_01,\n",
    "                svm_acc_01,\n",
    "                dnn_acc_01,\n",
    "                mlp_acc_01,\n",
    "                knn_acc_01,\n",
    "                cat_acc_01,\n",
    "                xgb_acc_01,\n",
    "                lgbm_acc_01,\n",
    "                rf_acc_01,\n",
    "                lr_acc_01,\n",
    "                voting_acc_01,\n",
    "                bag_svm_acc_01,\n",
    "                bag_knn_acc_01,\n",
    "                bag_dt_acc_01,\n",
    "                bag_lr_acc_01,\n",
    "                bag_mlp_acc_01,\n",
    "\n",
    "                bag_rf_acc_01,\n",
    "                bag_ada_acc_01,\n",
    "                bag_lgbm_acc_01,\n",
    "                # bag_xgb_acc_01,\n",
    "                bag_cat_acc_01,\n",
    "                bag_comb_acc_01,\n",
    "\n",
    "                avg_acc_01,\n",
    "                weighed_avg_acc_01\n",
    "                ]  \n",
    "\n",
    "\n",
    "level_01_pre = [\n",
    "                ada_pre_01,\n",
    "                svm_pre_01,\n",
    "                dnn_pre_01,\n",
    "                mlp_pre_01,\n",
    "                knn_pre_01,\n",
    "                cat_pre_01,\n",
    "                xgb_pre_01,\n",
    "                lgbm_pre_01,\n",
    "                rf_pre_01,\n",
    "                lr_pre_01,\n",
    "                voting_pre_01,\n",
    "                bag_svm_pre_01,\n",
    "                bag_knn_pre_01,\n",
    "                bag_dt_pre_01,\n",
    "                bag_lr_pre_01,\n",
    "                bag_mlp_pre_01,\n",
    "\n",
    "                bag_rf_pre_01,\n",
    "                bag_ada_pre_01,\n",
    "                bag_lgbm_pre_01,\n",
    "                # bag_xgb_pre_01,\n",
    "                bag_cat_pre_01,\n",
    "                bag_comb_pre_01,\n",
    "\n",
    "                avg_pre_01,\n",
    "                weighed_avg_pre_01\n",
    "                ]  \n",
    "\n",
    "level_01_rec = [\n",
    "                ada_rec_01,\n",
    "                svm_rec_01,\n",
    "                dnn_rec_01,\n",
    "                mlp_rec_01,\n",
    "                knn_rec_01,\n",
    "                cat_rec_01,\n",
    "                xgb_rec_01,\n",
    "                lgbm_rec_01,\n",
    "                rf_rec_01,\n",
    "                lr_rec_01,\n",
    "                voting_rec_01,\n",
    "                bag_svm_rec_01,\n",
    "                bag_knn_rec_01,\n",
    "                bag_dt_rec_01,\n",
    "                bag_lr_rec_01,\n",
    "                bag_mlp_rec_01,\n",
    "\n",
    "                bag_rf_rec_01,\n",
    "                bag_ada_rec_01,\n",
    "                bag_lgbm_rec_01,\n",
    "                # bag_xgb_rec_01,\n",
    "                bag_cat_rec_01,\n",
    "                bag_comb_rec_01,\n",
    "\n",
    "                avg_rec_01,\n",
    "                weighed_avg_rec_01\n",
    "                ]  \n",
    "\n",
    "level_01_f1 = [\n",
    "                ada_f1_01,\n",
    "                svm_f1_01,\n",
    "                dnn_f1_01,\n",
    "                mlp_f1_01,\n",
    "                knn_f1_01,\n",
    "                cat_f1_01,\n",
    "                xgb_f1_01,\n",
    "                lgbm_f1_01,\n",
    "                rf_f1_01,\n",
    "                lr_f1_01,\n",
    "                voting_f1_01,\n",
    "                bag_svm_f1_01,\n",
    "                bag_knn_f1_01,\n",
    "                bag_dt_f1_01,\n",
    "                bag_lr_f1_01,\n",
    "                bag_mlp_f1_01,\n",
    "\n",
    "                bag_rf_f1_01,\n",
    "                bag_ada_f1_01,\n",
    "                bag_lgbm_f1_01,\n",
    "                # bag_xgb_f1_01,\n",
    "                bag_cat_f1_01,\n",
    "                bag_comb_f1_01,\n",
    "\n",
    "                avg_f1_01,\n",
    "                weighed_avg_f1_01\n",
    "                ]  \n",
    "\n",
    "\n",
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_01_acc, level_01_pre, level_01_rec, level_01_f1))\n",
    "\n",
    "# Sort by F1-01 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[4], reverse=True)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_01_acc, sorted_level_01_pre, sorted_level_01_rec, sorted_level_01_f1 = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_01_acc[i]\n",
    "    data[i][2] = sorted_level_01_pre[i] \n",
    "    data[i][3] = sorted_level_01_rec[i] \n",
    "    data[i][4] = sorted_level_01_f1[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"ACC-01\", \"PRE-01\", \"REC-01\", \"F1-01\"]\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Summary table', file = f)\n",
    "if pick_prob == 1: \n",
    "    with open(output_file_name, \"a\") as f: print('Level 01 - Probabilities', file = f)\n",
    "else:\n",
    "    with open(output_file_name, \"a\") as f: print('Level 01 - CLASSES', file = f)\n",
    "if feature_selection_bit == 1: \n",
    "    with open(output_file_name, \"a\") as f: print('Feature Selection was applied', file = f)\n",
    "else:\n",
    "    with open(output_file_name, \"a\") as f: print('All features were used', file = f)\n",
    "\n",
    "\n",
    "    \n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+----------------+\n",
      "| Models      |   time-01(sec) |\n",
      "+=============+================+\n",
      "| avg         |      0.0400491 |\n",
      "+-------------+----------------+\n",
      "| KNN         |      0.0571139 |\n",
      "+-------------+----------------+\n",
      "| weighed_avg |      0.0625758 |\n",
      "+-------------+----------------+\n",
      "| Bag_knn     |      0.0923586 |\n",
      "+-------------+----------------+\n",
      "| SVM         |      0.095645  |\n",
      "+-------------+----------------+\n",
      "| LR          |      0.121755  |\n",
      "+-------------+----------------+\n",
      "| Bag_DT      |      0.128664  |\n",
      "+-------------+----------------+\n",
      "| Bag_svm     |      0.240201  |\n",
      "+-------------+----------------+\n",
      "| ADA         |      0.244629  |\n",
      "+-------------+----------------+\n",
      "| RF          |      0.289706  |\n",
      "+-------------+----------------+\n",
      "| XGB         |      0.319149  |\n",
      "+-------------+----------------+\n",
      "| LGBM        |      0.339041  |\n",
      "+-------------+----------------+\n",
      "| CAT         |      0.759877  |\n",
      "+-------------+----------------+\n",
      "| Bag_LR      |      0.788077  |\n",
      "+-------------+----------------+\n",
      "| Bag_ada     |      1.11689   |\n",
      "+-------------+----------------+\n",
      "| MLP         |      1.48823   |\n",
      "+-------------+----------------+\n",
      "| Bag_rf      |      1.57284   |\n",
      "+-------------+----------------+\n",
      "| Bag_lgbm    |      2.68354   |\n",
      "+-------------+----------------+\n",
      "| DNN         |      4.68657   |\n",
      "+-------------+----------------+\n",
      "| Bag_comb    |      6.0313    |\n",
      "+-------------+----------------+\n",
      "| Bag_cat     |      6.38573   |\n",
      "+-------------+----------------+\n",
      "| Bag_mlp     |     13.4463    |\n",
      "+-------------+----------------+\n",
      "| VOTING      |   9999         |\n",
      "+-------------+----------------+\n"
     ]
    }
   ],
   "source": [
    "# implement time table\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "                'avg',\n",
    "                'weighed_avg'\n",
    "                ]\n",
    "\n",
    "data = [[\"\" for _ in range(2)] for _ in range(len(names_models))]\n",
    "\n",
    "level_01_time = [\n",
    "                ada_time_01,\n",
    "                svm_time_01,\n",
    "                dnn_time_01,\n",
    "                mlp_time_01,\n",
    "                knn_time_01,\n",
    "                cat_time_01,\n",
    "                xgb_time_01,\n",
    "                lgbm_time_01,\n",
    "                rf_time_01,\n",
    "                lr_time_01,\n",
    "                voting_time_01,\n",
    "                bag_svm_time_01,\n",
    "                bag_knn_time_01,\n",
    "                bag_dt_time_01,\n",
    "                bag_lr_time_01,\n",
    "                bag_mlp_time_01,\n",
    "\n",
    "                bag_rf_time_01,\n",
    "                bag_ada_time_01,\n",
    "                bag_lgbm_time_01,\n",
    "                # bag_xgb_time_01,\n",
    "                bag_cat_time_01,\n",
    "                bag_comb_time_01,\n",
    "\n",
    "                avg_time_01,\n",
    "                weighed_avg_time_01\n",
    "                ]  \n",
    "\n",
    "\n",
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_01_time))\n",
    "\n",
    "# Sort by F1-01 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[1], reverse=False)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_01_time = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_01_time[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"time-01(sec)\"]\n",
    "\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Time is counted is seconds', file = f)\n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)\n",
    "end_program = time.time()\n",
    "time_program = end_program - start_program\n",
    "with open(output_file_name, \"a\") as f: print('Running time of entire program is:', time_program ,' seconds',file = f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ------------------------------------------------------------------"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
